kameleon-builder-2.11.0/0000755000004100000410000000000015005702337015044 5ustar www-datawww-datakameleon-builder-2.11.0/.editorconfig0000644000004100000410000000062415005702337017523 0ustar www-datawww-data# EditorConfig is awesome: http://EditorConfig.org # top-most EditorConfig file root = true [*] indent_style = space end_of_line = lf charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true [*.py] indent_size = 4 [*.yaml] trim_trailing_whitespace = false [*.{html,js,rb,scss,xml,less,css,rake,yaml}] indent_size = 2 [{Vagrantfile,Rakefile,Gemfile,Gemfile.lock}] indent_size = 2 kameleon-builder-2.11.0/bin/0000755000004100000410000000000015005702337015614 5ustar www-datawww-datakameleon-builder-2.11.0/bin/kameleon0000755000004100000410000000230715005702337017337 0ustar www-datawww-data#!/usr/bin/env ruby # Exit cleanly from an early interrupt Signal.trap("INT") { exit 1 } # Stdout/stderr should not buffer output $stdout.sync = true $stderr.sync = true require 'thor/error' require 'kameleon' # Force Thor to raise exceptions so we can exit non-zero. ENV["THOR_DEBUG"] = "1" begin Kameleon.init_userconf Kameleon::Main.start rescue Exception => e Kameleon.ui.error("Error: #{e}") if Kameleon.env.debug raise e else begin raise e rescue Kameleon::Exit => e exit e.status_code rescue Kameleon::Error => e Kameleon.ui.trace(e) exit e.status_code rescue Thor::UndefinedTaskError => e Kameleon.ui.trace(e) exit 15 rescue Thor::Error => e Kameleon.ui.trace(e) exit 15 rescue SystemExit, Interrupt => e Kameleon.ui.error("Quitting...") exit 1 rescue Errno::ENOENT => e Kameleon.ui.trace(e) exit 16 rescue Psych::SyntaxError => e Kameleon.ui.trace(e) exit 17 rescue Exception => e msg = "Unfortunately, a fatal error has occurred : "\ "#{e.message}.\nUse --debug option for more details\n" Kameleon.ui.error(msg) exit 666 end end end kameleon-builder-2.11.0/.gitignore0000644000004100000410000000042415005702337017034 0ustar www-datawww-data*~ *.swp *.gem *.rbc .bundle .config .yardoc Gemfile.lock InstalledFiles _yardoc coverage doc/ lib/bundler/man pkg rdoc spec/reports test/tmp test/version_tmp tmp coverage # Vagrant local workspace .vagrant *.log bin/* build share/python-wheels lib/python* pyvenv.cfg .envrc kameleon-builder-2.11.0/README.rst0000644000004100000410000000232715005702337016537 0ustar www-datawww-dataKameleon appliance builder ========================== Kameleon is a simple but powerful tool to generate customized operating system images, based on traceable recipes. Thanks to Kameleon, one can write recipes that describe how to create, step by step, customized operating systems in any desired target format, and then cook them (build them), just like GNU make cooks sources using a Makefile to build binary programs. For instance, Kameleon can create custom operating system images for QEMU/KVM, VirtualBox, docker, LXC or bootable ISO. It can support creating such images for any machine architecture (x86, ARM64, PPC64, ... ). In fact, since the Kameleon engine by itself is very generic by design, a lot more can be done, because most of the specialization happens in the recipes, written in Kameleon's powerful recipe language (YAML based DSL). Kameleon was initially developed to improve reproducibility in computer science and engineering, providing a tool that achieves complete *reconstructability* of system images with cache, checkpointing and interactive breakpoint mechanisms. * Latest documentation: http://kameleon.imag.fr/getting_started.html * Source code and issue tracker: https://github.com/oar-team/kameleon kameleon-builder-2.11.0/contrib/0000755000004100000410000000000015005702337016504 5ustar www-datawww-datakameleon-builder-2.11.0/contrib/kameleon_exec_cmd.sh0000644000004100000410000000122115005702337022456 0ustar www-datawww-data#!/usr/bin/env bash set -o pipefail set -o allexport __ROOT_DIRECTORY__=$(dirname $(readlink -f ${BASH_SOURCE[0]})) function save_env { # Save environment <% if Kameleon.env.debug %>set +x<% end %> (comm -3 <(declare | sort) <(declare -f | sort)) > "${__ROOT_DIRECTORY__}/<%= File.basename(@bash_env_file) %>" } trap 'save_env' INT TERM EXIT # Load environment source "${__ROOT_DIRECTORY__}/<%= File.basename(@bash_env_file) %>" 2> /dev/null || true # Log cmd echo <%= Shellwords.escape(cmd.value) %> >> "${__ROOT_DIRECTORY__}/<%= File.basename(@bash_history_file) %>" <% if Kameleon.env.debug %>set -o xtrace <% end %> <%= cmd.value %> kameleon-builder-2.11.0/contrib/kameleon_bashrc.sh0000644000004100000410000001363015005702337022160 0ustar www-datawww-data# If not running interactively, don't do anything export USER=${USER:-"root"} export HOME=${HOME:-"/root"} export PATH=/usr/bin:/usr/sbin:/bin:/sbin:$PATH export LC_ALL=${LC_ALL:-"POSIX"} export DEBIAN_FRONTEND=noninteractive mkdir -p $(dirname <%= @bash_history_file %>) ; touch "<%= @bash_history_file %>" mkdir -p $(dirname <%= @bash_env_file %>) ; touch "<%= @bash_env_file %>" mkdir -p $(dirname <%= @bash_status_file %>) ; touch "<%= @bash_status_file %>" source /etc/bash.bashrc 2> /dev/null export KAMELEON_CONTEXT_NAME="<%= @context_name %>_context" export HISTFILE="<%= @bash_history_file %>" ## functions function fail { echo $@ 1>&2 false } export -f fail ## aliases if [ -t 1 ] ; then # restore previous env source "<%= @bash_env_file %>" 2> /dev/null export TERM=xterm # for fast typing alias h='history' alias g='git status' alias l='ls -lah' alias ll='ls -lh' alias la='ls -Ah' # for human readable output alias ls='ls -h' alias df='df -h' alias du='du -h' # simple history browsing export HISTCONTROL=erasedups export HISTSIZE=10000 export HISTIGNORE="history*" shopt -s histappend bind '"\e[A"':history-search-backward bind '"\e[B"':history-search-forward # check the window size after each command and, if necessary, # update the values of LINES and COLUMNS. shopt -s checkwinsize # make less more friendly for non-text input files, see lesspipe(1) [ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" # If this is an xterm set the title to user@host:dir PROMPT_COMMAND='echo -ne "\033]0;${KAMELEON_CONTEXT_NAME:+($KAMELEON_CONTEXT_NAME)}${USER}@${HOSTNAME}: ${PWD}\007"' # set variable to show git branch when in a git repository # source: https://github.com/jimeh/git-aware-prompt/blob/master/prompt.sh # added highlighting of repo part in path function find_git_branch { git_subpath='/' local dir=${PWD} head until [ "$dir" = "" ]; do if [ -f "$dir/.git/HEAD" ]; then head=$(< "$dir/.git/HEAD") if [[ $head == ref:\ refs/heads/* ]]; then git_branch=" (${head#*/*/})" elif [[ $head != '' ]]; then git_describe=$(git describe --always) git_branch=" (detached: $git_describe)" else git_branch=' (unknown)' fi prompt_dir="${dir/$HOME/~}" return fi git_subpath="/${dir##*/}$git_subpath" dir="${dir%/*}" done git_branch='' prompt_dir="${PWD/$HOME/~}" git_subpath='' } function find_git_dirty { st=$(git status -s 2>/dev/null | tail -n 1) if [[ $st == "" ]]; then git_dirty='' else git_dirty='*' fi } export find_git_branch export find_git_dirty PROMPT_COMMAND="find_git_branch; find_git_dirty; history -a ; $PROMPT_COMMAND" # set a fancy prompt (non-color, unless we know we "want" color) case "$TERM" in xterm-color) color_prompt=yes;; esac # if the terminal has the capability; turned # off by default to not distract the user: the focus in a terminal window # should be on the output of commands, not on the prompt force_color_prompt=yes if [ -n "$force_color_prompt" ]; then if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then # We have color support; assume it's compliant with Ecma-48 # (ISO/IEC-6429). (Lack of such support is extremely rare, and such # a case would tend to support setf rather than setaf.) color_prompt=yes else color_prompt= fi fi if [ "$color_prompt" = yes ]; then PS1='${KAMELEON_CONTEXT_NAME:+($KAMELEON_CONTEXT_NAME) }\[\033[01;32m\]\u@\h\[\033[00m\]: \[\e[1;37m\]$prompt_dir\[\e[1;36m\]$git_subpath\[\e[0;31m\]$git_branch\[\e[1;33m\]$git_dirty\[\033[01;34m\] \$\[\033[00m\] ' else PS1='${KAMELEON_CONTEXT_NAME:+($KAMELEON_CONTEXT_NAME) }\u@\h: $prompt_dir$git_subpath$git_branch$git_dirty \$ ' fi # colors if [ -x /usr/bin/dircolors ]; then eval "`dircolors -b`" alias ls='ls --color=auto' #alias dir='dir --color=auto' #alias vdir='vdir --color=auto' alias grep='grep --color=auto' alias fgrep='fgrep --color=auto' alias egrep='egrep --color=auto' else alias ls='ls -G' fi fi function __download { echo "Downloading: $1..." if which curl >/dev/null; then curl -# -L --retry 999 --retry-max-time 0 "$1" -o "$2" 2>&1 else echo "curl is missing, trying with wget..." if which wget >/dev/null; then wget --retry-connrefused --progress=bar:force "$1" -O "$2" 2>&1 else echo "curl is missing, trying with python..." if which python >/dev/null; then python -c " import sys import time if sys.version_info >= (3,): import urllib.request as urllib else: import urllib def reporthook(count, block_size, total_size): global start_time if count == 0: start_time = time.time() return duration = time.time() - start_time progress_size = float(count * block_size) if duration != 0: if total_size == -1: total_size = block_size percent = 'Unknown size, ' else: percent = '%.0f%%, ' % float(count * block_size * 100 / total_size) speed = int(progress_size / (1024 * duration)) sys.stdout.write('\r%s%.2f MB, %d KB/s, %d seconds passed' % (percent, progress_size / (1024 * 1024), speed, duration)) sys.stdout.flush() urllib.urlretrieve('$1', '$2', reporthook=reporthook) print('\n') " true else fail "Cannot download $1" fi fi fi } export -f __download function __find_linux_boot_device() { local PDEVICE=`stat -c %04D /boot` for file in $(find /dev -type b 2>/dev/null) ; do local CURRENT_DEVICE=$(stat -c "%02t%02T" $file) if [ $CURRENT_DEVICE = $PDEVICE ]; then ROOTDEVICE="$file" break; fi done echo "$ROOTDEVICE" } export -f __find_linux_boot_device kameleon-builder-2.11.0/contrib/proxy_env.sh0000644000004100000410000000104615005702337021072 0ustar www-datawww-dataexport http_proxy="http://<%= @proxy %>" export https_proxy="http://<%= @proxy %>" export ftp_proxy="http://<%= @proxy %>" export rsync_proxy="http://<%= @proxy %>" export all_proxy="http://<%= @proxy %>" export HTTP_PROXY="http://<%= @proxy %>" export HTTPS_PROXY="http://<%= @proxy %>" export FTP_PROXY="http://<%= @proxy %>" export RSYNC_PROXY="http://<%= @proxy %>" export ALL_PROXY="http://<%= @proxy %>" export no_proxy="localhost,$(echo <%= @proxy %> | tr ":" "\n" | head -n 1),127.0.0.1,localaddress,.localdomain" export NO_PROXY="$no_proxy" kameleon-builder-2.11.0/contrib/kameleon_exec_cmd_wrapper.sh0000644000004100000410000000105215005702337024220 0ustar www-datawww-data#!/usr/bin/env bash ROOT_DIRECTORY=$(dirname $(readlink -f ${BASH_SOURCE[0]})) function post_exec_wrapper { echo $? > "$ROOT_DIRECTORY/<%= File.basename(@bash_status_file) %>" # Print end flags echo -n <%= cmd.end_err %> 1>&2 echo -n <%= cmd.end_out %> } function pre_exec_wrapper { # Print begin flags echo -n <%= cmd.begin_err %> 1>&2 echo -n <%= cmd.begin_out %> } trap 'post_exec_wrapper' INT TERM EXIT ## Started pre_exec_wrapper bash --rcfile "<%= @bashrc_file %>" <%= File.join(@bash_scripts_dir, "#{cmd}.sh" ) %> kameleon-builder-2.11.0/CHANGES0000644000004100000410000005232415005702337016045 0ustar www-datawww-dataKameleon CHANGELOG ================== Version 2.10.14 → 2.11.0 ------------------------ Released on May 4th 2025 - Enable command aliases in the definition of checkpoint actions. - Enable command aliases with no args. - Fix dependencies to progressbar and ruby-graphviz (the "ruby-" prefix matters!). - Fix terminal width issue with latest Thor. - Fix graphviz and progress bar dependencies. - Rework README.rst and the documentation front page. - Some other minor changes. Version 2.10.13 --------------- Released on March 26th 2025 - Drop useless contrib directory. - Rework gem summary and description. - Update gem dependencies for gem install on Debian 11 and 13. - Adapt sources to Ruby 3.1 - Psych's YAML: explicitly use YAML.unsafe_load. - Add git to requirements, need for kameleon repo commands. Version 2.10.12 --------------- Released on February 18th 2025 - Fix AUTHORS: update maintainer. - Fix bug with File.exists? (removed from ruby 3.3), use File.exist? - Fix gemspec: License and update Thor to remove the DidYouMean::SPELL_CHECKERS warning. Version 2.10.11 --------------- Released on October 10th 2023 - Rework the checkpointing code. - Make build --from-checkpoint not activate the checkpointing without the --enable-checkpointing option. - Add the -F alias for the build --from-checkpoint option and allow passing the microstrep slug instead of it's id. - Add the build --begin-checkpoint (or -B) and --end-checkpoint (or -E) options that allows giving the window of steps to checkpoint. - Remove the dryrun command and add the --dryrun (or -d) switch to the build action. - Make build --dryrun --list-checkpoints (or -d -l) to show possible checkpoints for the recipe. Version 2.10.10 --------------- Released on June 13th 2023 - Set kameleon build --microstep-checkpoint option to "all" by default. - Rework the kameleon checkpoint documentation and more. - Rework the kameleon build options. - Add the on_checkpoint=only keyword, allowing a step to be run only when checkpoint is activated. - Add the on_checkpoint=disabled keyword, allowing a step to be never be checkpointed. - Add the $${checkpointing_enabled} global variable, value is "true" or "false" whether checkpointing is enabled or not. - Fix error in kameleon info: variable not found $${checkpointing_enabled} / $${persistent_cache}. - Make the context_reload command less verbose by default, and some other commands by side effects. - Other cosmetic fixes. Version 2.10.9 -------------- Released on April 27th 2023 - Add the --microstep-checkpoint option to enable creating checkpoint on all microsteps. - Add timing info for microsteps and microstep checkpoints in verbose mode. - Minor fixes. - Enable using a python venv for bumpversion (.gitignore, .envrc). Version 2.10.8 -------------- Released on October 21st 2022 - Fix a fatal error with the pipe command: make sure the destination context is started. - Avoid git warning when running kameleon repo update: use git pull --ff-only. Version 2.10.7 -------------- Released on December 02nd 2021 - Add support for the .filter file to filter recipes/template lists. Version 2.10.6 -------------- Released on November 24th 2021 - Add the option filter for the template and recipe listings. - Preserve permission of files when importing template. Version 2.10.5 -------------- Released on July 29th 2021 - Remove polipo from the software dependencies as it is not maintained anymore. This breaks the caching feature. - Some changes to allow a debian packaging. - Update authors. - Change the version information file and update scripts accordingly. Version 2.10.4 -------------- Released on May 11th 2020 - Fix support for extend ERB. - Add the `kameleon template erb` command. - Fix bash completion. - Cosmetic code fixes. Version 2.10.3 -------------- Released on April 10th 2020 - Rework kameleon template list: add color add progress bar. - Make bash completion understand the subcommands. - Fix the command help -> `kameleon -h`. - Add support for custom extend erb templates. Version 2.10.2 -------------- Released on April 09th 2020 - Fix cli help for the repository and template sub-commands. - Add the git remote url and branch to kameleon repo list. - Add the 'kameleon repository remove' command. Version 2.10.1 ------------- Released on March 21th 2020 - Fix regression in 2.10.0 with the build directory creation. Version 2.10.0 ------------- Released on March 21th 2020 - Rework usage (cosmetic fixes). - Drop the `kameleon template repository` action (same as `kameleon repository`). - Make `kameleon dag` and `kameleon dryrun` standalone actions instead of `kameleon info` options. - Add the `kamelon dag --recipes-only`. - Make `kameleon dag` show distincly the extended recipe among the ancestors. - Add the `kameleon export` action. Version 2.9.4 ------------- Released on December 12th 2018 - Minor fixes. - NB: the 2.9.3 version was lost in space... Version 2.9.2 ------------- Released on February 21st 2018 - Enforce the number of sub commands for the rescue and test commands. - Support nested aliases. Version 2.9.1 ------------- Released on January 28st 2018 - Make the test and group command work in the cleaning steps. Version 2.9.0 ------------- Released on June 29th 2017 - Bump to a new minor version due to the addition of the test and group commands. - Refresh documentation. Version 2.8.4 ------------- - Fix the build usage message. - Fix the cache compression options: allow no compression. - Add the test and group commands for microsteps. Version 2.8.3 ------------- - Patch the doc and the default new recipe template. Version 2.8.2 ------------- Released on September 06th 2016 - Make kameleon new and template import works. Version 2.8.1 ------------- - Manage problem with gem update. Version 2.8.0 ------------- Released on September 05th 2016 - Add the possibility to overload extend with backend. Version 2.7.8 ------------- - Fix info data resolution (fix #78). Version 2.7.7 ------------- Released on September 01st 2016 - Add some warning to avoid global CLI option misuse. Version 2.7.6 ------------- Released on August 31st 2016 - Fix step variable resolution for composed variable (#79), and containing. uuid (#90). Version 2.7.5 ------------- Released on August 31st 2016 - Fix last problem with Kameleon_data_dir (again). Version 2.7.4 ------------- Released on August 29th 2016 - Fix Kameleon_data_dir is not rendering correctly with {} (#85). Version 2.7.3 ------------- Released on June 20th 2016 - The info command can now handle multiple recipes. - Colorized the output of the info command. - Added the dryrun option to the info command. - Added the dag option to the info command: draw a GraphViz dag for recipes. Version 2.7.2 ------------- Released on February 17th 2016 - Added ``interactive_cmd`` option to set a more apropriate interactive shell command. - Removed bash errexit flag to force bash to trap interruption. Version 2.7.1 ------------- - Non-strictly template variable resolution (Fixed #67). Version 2.7.0 ------------- Released on December 09th 2015 - Fixed variables overload when using inheritance. - Added support for including global options from a separate file (#57). - Allowed kameleon to fail silently if some error occured during include. - Marked all bash variables for export. - Sorted variables in ``kameleon info`` dump. - Removed duplicate error messages. - Shutdown polipo cleanly (#54). - Added offline mode to prevent Polipo from contacting remote servers. - Fixed polipo default options to avoid "206 partial responses" (#54). - Made only one checkpoint per macrostep to improve performance. - Made polipo log file unique for each user to avoid "Permission denied" error. - Added new keyword in ZSH completion. Version 2.6.7 ------------- Released on October 1st 2015 - Decreased polipo exit timeout (120 to 5s). - Passed only recipe files (not directories) to the cache to fixed persistant cache crash. Version 2.6.6 ------------- Released on September 14th 2015 - set default ``cache_archive_compression`` to gzip. Version 2.6.5 ------------- Released on September 14th 2015 - Fixed the proxy-cache to allow big files. - Added ``--proxy`` and ``--proxy-credentials`` options to set a global proxy for kameleon. - Added ``proxy_local``, ``proxy_out`` and ``proxy_in`` kameleon recipe variables to get proxy address from recipe. - Enabled ERB templating in the user environement files. - Renamed ``--proxy-path`` option to ``--polipo-path``. Version 2.6.4 ------------- Released on September 7th 2015 - Fixed kameleon data variables resolution. Version 2.6.3 ------------- Released on August 25th 2015 - Don't fail now if a context was closed. Version 2.6.2 ------------- Released on August 25th 2015 - Added ``cache_archive_compression`` option to choose compression tool. - Tried using curl before wget. Version 2.6.1 ------------- Released on August 12th 2015 - Cli global parameters overwrite all recipe key:value parameters. Version 2.6.0 ------------- Released on May 13th 2015 - Added --list-checkpoints to ``kameleon build`` command. - Removed ``kameleon checkpoints`` and ``kameleon clean`` commands. - Displayed error information in normal and verbose mode. - Fixed ruby 1.8.7 compatibility. - Added Zsh completion (See completion). Version 2.5.0 ------------- Released on Jan 27th 2015 - Allowed user to get custom shell environement by adding shell script. - Added option ``--verbose`` to allow user debugging. - Added option ``--from-cache`` to the ``kameleon info`` subcommand. - Enabled bash xtrace in verbose mode. - Checked appropriate proxy cache server start. - Fixed recipe path in proxy cache metadata. - Forced proxy cache to use IPv4. - Improved UI with more color and less output messages. Version 2.4.0 ------------- Released on Dec 23rd 2014 - Supported recipe attached data. - Introduced a new shell command execution mechanism, to prevent kameleon from waiting forever if the command act weird (Fixed #39). - Improved the customization of steps with inheritance. - Allowed to set global variables from cli during build using ``--global key:value ..``. - Moved repository command to the upper level (``kameleon repository``). Version 2.3.5 ------------- Released on Dec 2nd 2014 - Fixed kameleon 2.3.4 regressions about ``kameleon build``. Version 2.3.4 ------------- Released on Nov 26th 2014 - Fixed kameleon 2.3.3 regressions about the template import. Version 2.3.3 ------------- Released on Nov 26th 2014 - Minor improvements. - Fixed issue with listing the templates avaiable in a repository. Version 2.3.2 ------------- Released on Nov 20th 2014 - Search steps directories in the workspace first. Version 2.3.1 ------------- Released on Nov 5th 2014 - ``--from-checkpoint`` option automatically enable the checkpoint process. - ``--from-checkpoint`` could take a special value "last" to refer to the last valid checkpoint. Version 2.3.0 ------------- Released on Oct 16th 2014 - Handled Ctrl-C to make a breakpoint instead of quitting. (Fixed #29). - Stored kameleon user configuration file in ``~/.kameleon.d/config`` (Fixed #24). - Fixed ``kameleon new`` to take an absolute or relative path and place the recipe in a subdir (Fixed #22). - Moved builtin recipes to an external repository: See [http://kameleon.imag.fr/repository.html](Docs) (Fixed #24). - Renamed ``--cache`` option to ``--enable-cache``. - Renamed ``--checkpoint`` option to ``--enable-checkpoint``. - Removed ``kameleon import`` command. - Removed ``kameleon templates`` command. - Added ``kameleon info`` to display detailed information about a recipe. - Added ``kameleon list`` to list all defined recipes in the current directory. - Added ``kameleon template info`` to display detailed information about a template. - Added ``kameleon template import`` to import a template. - Added ``kameleon template list`` to list all available templates. - Added ``kameleon template repository`` to manage external git recipes repositories. Version 2.2.5 ------------- Released on Sep 19th 2014 - [template] Customized vm memory size before vagrant export. - [template] Configured virtualbox to use ``82540EM`` driver for ethX interfaces. - [template][debian] installed man-db and net-tools with debootstrap. - [template][centos] Removed traces of mac address from network configuration. - [template][centos] installed man package. - [template][centos] Set OS type to ``Redhat_64`` in virtualbox. - [template][debian] Added security and backports repository. - [template][debian] Added debian security and backports repositories. - [template][debian] Removed ntp from all debian recipes. - [template][centos] Added ``single-request-reopen`` option to network config (centos). - [template][centos] Fixed corrupted rpm database. Version 2.2.4 ------------- Released on Sep 2nd 2014 - [template] Added the ``-cpu host`` option to qemu to improve performances. - [template] Made centos image as close as possible as default Centos installation. - [template] Configured SELinux for Fedora and Centos. - [template] Fixed Grid'5000 export step. - [template] Extended vagrant recipes from virtualbox recipes. - [core] Adding the step elapsed time to the Kamelon output. - [proxy cache] Restructured persistent cache and improved caching of pipes. - [proxy cache] Added ``ProxyAdrres`` paramter to polipo to take into account all the host network interfaces. Version 2.2.3 ------------- Released on Aug 19th 2014 - [template] The extlinux MBR path has changed because jessie is not frozen (Fix #17). Version 2.2.2 ------------- Released on July 23rd 2014 - [core] Added ``-h`` CLI option to print help. - [core] Added ``--templates-path`` CLI option to set different templates directory. - [core] Fixed regression about recipe ancestors loading. Version 2.2.1 ------------- Released on July 22nd 2014 - [core] Switch to default yaml parser (psych) and removed syck from dependencies. - [core] Loaded a recipe with all ancestors. Version 2.2.0 ------------- Released on July 20th 2014 - [proxy cache] Added ``proxy_cache`` recipe option to context_in/out to enable persistant cache for every context. - [proxy cache] Added ``--cache-path`` CLI option to set different cache directory. - [proxy cache] Shared cache directory between recipes by default. - [proxy cache] No more internal cache for debootstrap. - [proxy_cache] Polipo does not use default config file anymore. - [proxy_cache] Recipe is now stored in cache at the end. - [package] Removed diffy from dependencies. - [package] Removed log4r-color from dependencies. - [core] Rewrote checkpoint mechanism. (allow to run multiple commands). - [core] Added ``--script`` CLI option to enable non-interactive mode. - [core] Added ``reload_context`` command that reloads the context from the recipe. - [core] From now on, Kameleon tries to clean all contexts without blocking the shell. - [core] Improved cleaning contexts upon interruption. - [core] If microstep is marked as ``redo`` or ``skip`` (on checkpoint), the ID will be independent from other steps. - [template] Grouped templates by virtualization tools. - [template] Removed extlinux timeout for debian. - [template] Configured apt to install recommended packages by default. - [template] Added architecture type in templates description. - [template] Forced killing qemu with sigterm signal. - [template] Removed ssh ControlPath option. - [template] Used parted tool in script mode. - [template] Added a timeout during startup and shutdown VMs. - [template] Restored eth0 as default interface in Fedora (net.ifnames=0 biosdevname=0). - [template] Ensured that the checkpoint was disabled at the beginning of the recipe. - [template] Make sure that qemu is stopped at the end. - [template] Replaced netcat by socat. - [template] Removed duplicated ubuntu repository (restricted). - [template] Used a debian-jessie iso to bootstrap the out_context with a minimal linux system. - [template] Hide default kameleon state files. - [template] Enabled debug mode with ``KAMELEON_DEBUG=1`` environment variable. - [template] Added ``download_file`` alias that uses curl/wget or python to download files. - [template] Enabled kvm only if available. - [template] Do not run qemu with nohup (Use ``-daemonize`` instead). - [template] Improved context_out/in log readability. - [template] Removed fedora-rawhide templates. - [template] Added new templates: - qemu/archlinux-desktop-i686 - qemu/archlinux-i686 - qemu/centos6.5-x86_64 - qemu/centos7-x86_64 - qemu/debian8-amd64 - vagrant/debian7-amd64 - virtualbox/archlinux-desktop-i686 - virtualbox/archlinux-desktop-x86_64 - virtualbox/archlinux-i686 - virtualbox/archlinux-x86_64 - virtualbox/centos6.5-i386 - virtualbox/centos6.5-x86_64 - virtualbox/centos7-x86_64 - virtualbox/debian7-amd64 - virtualbox/debian7-desktop-amd64 - virtualbox/debian7-i386 - virtualbox/debian7-kameleon-amd64 - virtualbox/debian7-oar-amd64 - virtualbox/debian8-amd64 - virtualbox/debian8-i386 - virtualbox/fedora20-x86_64 - virtualbox/ubuntu-12.04-amd64 - virtualbox/ubuntu-12.04-desktop-amd64 - virtualbox/ubuntu-14.04-amd64 - virtualbox/ubuntu-14.04-desktop-amd64 Version 2.1.3 ------------- Released on June 16th 2014 - [core] Loading dependent gems based on the users ruby version. - [core] Getting back compatibility with ruby 1.8.7. - [core] Don't fail if context is not ready. - [core] Minor bug fixes. - [template] Set a small timeout with netcat. - [template] Added new debian 7 appliance with kameleon. Version 2.1.0 ------------- Released on June 12th 2014 - [core] Fixed psych yaml parsing (#1). - [core] Changed option ``--no-no-color`` to ``--color``. - [core] Saved the contexts state files in their WORKDIR (#3). - [core] Set context in/out/local cmd to /bin/bash by default (#5). - [core] Made global section non mandatory. - [core] Made writing embedded step in recipe possible (#12). - [core] Improved the readability of logs and the progress bar. - [core] Moved aliases and checkpoints folders to steps. - [core] Removed the ``recipes`` folder and the ``workspace`` (#2). - [core] Make a safe copy with ``kameleon new`` command. - [core] Added a simple extend recipe feature (#11). - [core] Introduced the keyword "@base" in the extended recipes (#11). - [core] Don't log identifier of microstep during build process. - [core] Added ``kameleon import`` command (#11). - [core] Added ``--clean`` option to ``kameleon build`` command. - [core] Added the lazy context initialization (#10). - [core] Set the variable ``KAMELEON_WORKDIR`` for all contexts. - [core] Used ``KAMELEON_WORKDIR`` when working with PIPE. - [core] Added persistent cache feature to Kameleon, So far it is caching just packages comming from the network using Polipo. - [template] Added new templates: - archlinux - archlinux-desktop - debian-testing - debian7 - debian7-desktop - debian7-oar-dev - fedora-rawhide - fedora20 - fedora20-desktop - ubuntu-12.04 - ubuntu-12.04-desktop - ubuntu-14.04 - ubuntu-14.04-desktop - vagrant-debian7 - [template] Installed the extlinux bootloader depending on distributions. - [template] New way to bootstrap fedora using Liveos image. - [template] Installed linux kernel and extlinux bootloader from bootstrap section. - [template] Used parted instead of sfdisk. - [template] Added save_as_qed step. - [template] Removed insecure ssh key before any export. - [template] Added shell auto-completion for bash, zsh and fish shell. - [template] Default user group is sudo. - [template] Added a new qemu/kvm template with full-snapshot support. - [template] Ability to add user in multiple groups (with usermod -G). - [template] Improved I/O performance with qemu/kvm. - [template] Removed force-unsafe-io for dpkg to avoid corrupted filesystem. - [template] Used qemu by default instead of chroot. - [template] Added option to disable debootstrap cache. - [template] Refactor qcow2 backing file checkpoints. - [template] Make QEMU checkpoint more robust and avoid disk corruption. - [template] Major revision of steps to make it easier to use in different templates. - [template] Rename steps for more semantic consistency. - [template] Making the 'save_appliance' step not dependent on the state of the machine. - [template] Enabled cache for arch_bootstrap. - [template] Added openssh in arch-bootstrap and enabled sshd.service/dhcp.service. - [template] Added user 'nobody' to allow sshd to run in the archlinux virtual machine. - [template] Enabled checkpoints (backing-file) only in the "setup" stage. - [template] Fixed .ssh and authorized_keys permissions. - [template] Avoid crash of in_context when we send a shutdown command to the virtual machine. - [template] Exclude special files with rsync (proc/dev...) when copying rootfs to the disk. - [template] Force stop qemu if still running. - [template] Make debian-chroot depreciated. - [template] Refactor archlinux template to use it with qemu/kvm. - [template] Improved the LiveOS fedora bootstrap step to get the system running with qemu/kvm. - [template] Refactor fedora20/debian8 templates to use them with qemu/kvm. - [template] Set timezone to UTC by default. - [template] Used ProxyCommand to improve the debian7-g5k recipe. - [aliases] Updated write_file and append_file aliases to support double quotes. - [aliases] Defined new aliases for unmounting devices. - [docs] More documentation. Version 2.0.0 ------------- Released on February 17th 2014 - Initial public release of kameleon 2. kameleon-builder-2.11.0/kameleon-builder.gemspec0000644000004100000410000000275415005702337021640 0ustar www-datawww-data# coding: utf-8 lib = File.expand_path('lib', __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require 'kameleon/version' Gem::Specification.new do |s| s.name = 'kameleon-builder' s.version = Kameleon::VERSION s.date = Time.now.strftime('%Y-%m-%d') s.authors = ['Salem Harrache', 'Michael Mercier', 'Cristan Ruiz', 'Pierre Neyron', 'Bruno Bzeznik'] s.email = ['salem@harrache.info', 'michael.mercier@libr.fr', 'camilo1729@gmail.com', 'pierre.neyron@imag.fr', 'bruno.bzeznik@imag.fr'] s.summary = 'The mindful appliance builder' s.description = 'Kameleon is a tool to build system appliances, possibly from scratch' s.homepage = 'http://kameleon.imag.fr/' s.license = 'GPL-2.0-or-later' s.files = `git ls-files`.split($/) s.files.reject! { |file| file.start_with?('docs/') } s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) } s.test_files = s.files.grep(%r{^(tests|s|features)/}) s.require_paths = ['lib'] s.add_dependency 'childprocess', '~> 4.0' s.add_dependency 'progressbar', '~> 1.10' s.add_dependency 'psych', '~> 5.0' s.add_dependency 'ruby-graphviz', '~> 1.2' s.add_dependency 'table_print', '~> 1.5' s.add_dependency 'thor', '~> 1.0' s.requirements << 'git' s.requirements << 'graphviz' end kameleon-builder-2.11.0/AUTHORS0000644000004100000410000000146115005702337016116 0ustar www-datawww-data Kameleon is developed within the Laboratoire LIG Batiment IMAG 150 Place du Torrent 38400 Saint Martin d'Hères FRANCE Kameleon is a sub-project of OAR, and so maintained by the OAR team. Initially developed by: Darko Illic < darko.ilic@gmail.com > Joseph Emeras < Joseph.Emeras@imag.fr > Olivier Richard < olivier.richard@imag.fr > Philippe Le Brouster < philippe.le-brouster@imag.fr > Salem Harrache < salem.harrache@inria.fr > Michael Mercier < michael.mercier@inria.fr > Cristan Ruiz < cristian.ruiz@imag.fr > Bruno Bzeznik < bruno.bzeznik@imag.fr > Now maintained by: Pierre Neyron < pierre.neyron@imag.fr > kameleon-builder-2.11.0/lib/0000755000004100000410000000000015005702337015612 5ustar www-datawww-datakameleon-builder-2.11.0/lib/kameleon.rb0000644000004100000410000000545215005702337017740 0ustar www-datawww-datarequire 'fileutils' require 'optparse' require 'erb' require 'ostruct' require 'tempfile' require 'pp' require 'thor' require 'childprocess' require 'pathname' require 'table_print' require 'yaml' require 'find' require 'kameleon/ui' module Kameleon class << self attr_writer :env attr_writer :ui attr_writer :source_root attr_writer :log_on_progress attr_writer :userdir attr_writer :userconf_path attr_writer :repositories_path attr_writer :default_values # The source root is the path to the root directory of the kameleon gem. def source_root @source_root ||= Pathname.new(File.expand_path('../../', __FILE__)) end def erb_dirpath d = File.join(Kameleon.source_root, 'erb') if File::directory?(d) return d elsif File::directory?(d = '/usr/share/kameleon/erb') return d end end def userdir @userdir ||= Pathname.new(File.join('~', '.kameleon.d')) Dir.mkdir(File.expand_path(@userdir.to_s)) unless File.exist?(File.expand_path(@userdir.to_s)) @userdir end def userconf_path @userconf_path ||= Pathname.new(File.join(File.expand_path(userdir.to_s), 'config')) end def init_userconf() if not File.exist?(Kameleon.userconf_path) or File.zero?(Kameleon.userconf_path) File.open(Kameleon.userconf_path, 'w+') do |file| userconf_erb = File.join(Kameleon.erb_dirpath, "userconf.yaml.erb") erb = ERB.new(File.open(userconf_erb, 'rb') { |f| f.read }) result = erb.result(binding) file.write(result) end end end def load_userconf if File.exist?(Kameleon.userconf_path) and not File.zero?(Kameleon.userconf_path) yaml_conf = YAML.unsafe_load_file Kameleon.userconf_path unless yaml_conf.kind_of? Hash yaml_conf = {} end else yaml_conf = {} end return yaml_conf end def default_values userconf = load_userconf @default_values ||= { :color => userconf.fetch("color", true), :debug => userconf.fetch("debug", false), :script => userconf.fetch("script", false), :repositories_path => userconf.fetch("repositories_path", File.join(userdir.to_s, 'repos')), :extend_yaml_erb => userconf.fetch("extend_yaml_erb", ".extend.yaml.erb") } end def env @env ||= Environment.new end def ui @ui ||= UI::Shell.new end def log_on_progress @log_on_progress ||= false end end end # Load the things which must be loaded before anything else require 'kameleon/compat' require 'kameleon/utils' require 'kameleon/error' require 'kameleon/repository' require 'kameleon/cli' require 'kameleon/environment' require 'kameleon/version' kameleon-builder-2.11.0/lib/kameleon/0000755000004100000410000000000015005702337017405 5ustar www-datawww-datakameleon-builder-2.11.0/lib/kameleon/persistent_cache.rb0000644000004100000410000002440715005702337023264 0ustar www-datawww-datarequire 'childprocess' require 'singleton' require 'socket' require 'net/http' module Kameleon #This ruby class will control the execution of Polipo web proxy class Persistent_cache include Singleton attr_reader :cache_dir attr_reader :polipo_port attr_writer :activated attr_reader :cwd attr_writer :polipo_path attr_reader :name attr_writer :cache_path attr_accessor :mode attr_accessor :name attr_accessor :recipe_files # FIXME have to check those. attr_accessor :recipe_path attr_accessor :archive_format attr_accessor :polipo_cmd_options attr_accessor :offline def initialize() ## we must configure Polipo to be execute for the in and out context ## we have to start polipo in the out context for debootstrap step @polipo_process = nil @polipo_port = find_unused_port @polipo_cmd_options = {:diskCacheRoot => "", :maxDiskCacheEntrySize => "-1", :idleTime => "1", :allowedClients => "0.0.0.0/0", :proxyPort => @polipo_port, :daemonise => false, :proxyAddress => "0.0.0.0", :logFile => File.join(Kameleon.env.build_path, "polipo-#{ENV['USER']}.log"), :logLevel => "0xFF", } @activated = false @mode = nil #It could be build or from @cache_dir = Kameleon.env.cache_path @polipo_path = nil @cwd = "" @cmd_cached = [] @cache_path = "" @current_raw_cmd = nil @current_step_dir = nil @recipe_file = nil @steps_files = [] @cached_recipe_dir = nil @offline = false end def find_unused_port ports = (8000..9000) port = 0 tmp = nil ports.each do |p| begin port = p tmp = TCPServer.new('127.0.0.1', port) rescue port =0 end break if(port>0) end tmp.close port end def check_polipo_binary @polipo_path ||= Utils.which("polipo") if @polipo_path.nil? then Kameleon.ui.error("Polipo binary not found, make sure it is in your current PATH") Kameleon.ui.error("or use the option --proxy-path") raise BuildError, "Failed to use persistent cache" end end def activated? @activated end def cwd=(dir) @cwd = dir end def create_cache_directory(step_name) Kameleon.ui.debug("Creating cache directory #{step_name} for Polipo") directory_name = File.join(@cache_dir,"DATA","#{step_name}") FileUtils.mkdir_p directory_name directory_name end def proxy_is_running?() begin res = Net::HTTP.get_response(URI("http://127.0.0.1:#{@polipo_port}/polipo/status")) if not @offline if not res.body.include? "is on line" Kameleon.ui.debug("The proxy is running but not responding. Server response: #{res.inspect}") return false else Kameleon.ui.debug("The proxy is responding") return true end end return true rescue Exception => e Kameleon.ui.debug("The proxy is not responding. Server response: #{e.message}") return false end end def start_web_proxy_in(directory) ## This function assumes that the cache directory has already been created by the engine # setting current step dir @current_step_dir = directory Kameleon.ui.info("Starting web proxy Polipo in directory #{directory} using port: #{@polipo_port}") @polipo_process.stop(0) unless @polipo_process.nil? command = ["#{@polipo_path}/polipo", "-c", "/dev/null"] @polipo_cmd_options[:diskCacheRoot] = directory @polipo_cmd_options.each{ |v,k| command.push("#{v}=#{k}") } ChildProcess.posix_spawn = true Kameleon.ui.debug("Starting process '#{command}'") @polipo_process = ChildProcess.build(*command) @polipo_process.start timeout = 0 while ( not(proxy_is_running?) and timeout < 5 ) sleep 1 timeout = timeout + 1 end return (@polipo_process.alive? and proxy_is_running?) end def stop_web_proxy # http://www.pps.univ-paris-diderot.fr/~jch/software/polipo/manual/Stopping.html Kameleon.ui.info("Stopping web proxy polipo") unless (@polipo_process.nil? || @polipo_process.exited?) Process.kill("USR1", @polipo_process.pid) # will write out all the in-memory data to disk Process.kill("SIGINT", @polipo_process.pid) # will shut down cleanly begin @polipo_process.poll_for_exit(10) rescue ChildProcess::TimeoutError @polipo_process.stop # tries increasingly harsher methods to kill the process. end end end def pack() if @archive_format.eql? "gzip" env = {"GZIP" => "-9"} Kameleon.ui.info("Packing up the generated cache in #{@cwd}/#{@name}-cache.tar.gz") execute("tar","-czvf #{@name}-cache.tar.gz -C #{@cache_dir} .", @cwd, env) elsif @archive_format.eql? "bz2" env = {"BZIP" => "-9"} Kameleon.ui.info("Packing up the generated cache in #{@cwd}/#{@name}-cache.tar.bz2") execute("tar","-cjvf #{@name}-cache.tar.bz2 -C #{@cache_dir} .", @cwd, env) elsif @archive_format.eql? "xz" env = {"XZ_OPT" => "-9"} Kameleon.ui.info("Packing up the generated cache in #{@cwd}/#{@name}-cache.tar.xz") execute("tar","-cJvf #{@name}-cache.tar.xz -C #{@cache_dir} .", @cwd, env) else env = {} Kameleon.ui.info("Packing up the generated cache in #{@cwd}/#{@name}-cache.tar") execute("tar","-cvf #{@name}-cache.tar -C #{@cache_dir} .", @cwd, env) end end def unpack(cache_path) Kameleon.ui.info("Unpacking persistent cache: #{cache_path}") FileUtils.mkdir_p @cache_dir execute("tar","-xf #{cache_path} -C #{@cache_dir}") end # This function caches the raw command specified in the recipe def cache_cmd_raw(raw_cmd_id) @current_raw_cmd = raw_cmd_id return true end def cache_cmd(cmd,file_path) Kameleon.ui.info("Caching file") Kameleon.ui.debug("command: cp #{file_path} #{@cwd}/cache/files/") FileUtils.mkdir_p @current_step_dir + "/data/" FileUtils.cp file_path, @current_step_dir + "/data/" @cmd_cached.push({:raw_cmd_id => @current_raw_cmd, :stdout_filename => File.basename(file_path)}) end def get_cache_cmd(cmd) return false if @mode == :build cache_line = @cmd_cached.select{ |reg| reg[:raw_cmd_id] == @current_raw_cmd }.first if cache_line.nil? then # This error can be due to the improper format of the file cache_cmd_index Kameleon.ui.error("Persistent cache missing file") raise BuildError, "Failed to use persistent cache" end return File.new("#{@current_step_dir}/data/#{cache_line[:stdout_filename]}","r") end def stop() stop_web_proxy Kameleon.ui.info("Finishing persistent cache with last files") cache_metadata_dir = File.join(@cache_dir,"metadata") if @mode == :build then File.open("#{cache_metadata_dir}/cache_cmd_index",'w+') do |f| f.puts(@cmd_cached.to_yaml) end unless @recipe_files.empty? all_files = @recipe_files.push(@recipe_path) recipe_dir = Pathname.new(common_prefix(all_files)) cached_recipe_dir = Pathname.new(File.join(@cache_dir,"recipe")) Kameleon::Utils.copy_files(recipe_dir, cached_recipe_dir, all_files) end ## Saving metadata information Kameleon.ui.info("Caching recipe") File.open("#{cache_metadata_dir}/header",'w+') do |f| if recipe_dir.nil? recipe_path = @recipe_path.basename else recipe_path = @recipe_path.relative_path_from(recipe_dir) end f.puts({:recipe_path => recipe_path.to_s}.to_yaml) f.puts({:date => Time.now.to_i}.to_yaml) end #Removing empty directories cache_data_dir = File.join(@cache_dir,"DATA") Dir.foreach(cache_data_dir) do |item| dir_temp = File.join(cache_data_dir,item) Dir.delete(dir_temp) if File.stat(dir_temp).nlink == 2 end pack end end def start() check_polipo_binary if @mode == :from then begin unpack(@cache_path) rescue raise BuildError, "Failed to untar the persistent cache file" end ## We have to load the file metadata_dir = File.join(@cache_dir,"metadata") @cmd_cached = YAML.unsafe_load(File.read("#{metadata_dir}/cache_cmd_index")) end @activated = true #@cached_recipe_dir = @cache_dir FileUtils.mkdir_p @cache_dir # Creating sctructure of the cache FileUtils.mkdir_p File.join(@cache_dir,"recipe") FileUtils.mkdir_p File.join(@cache_dir,"DATA") FileUtils.mkdir_p File.join(@cache_dir,"metadata") @polipo_cmd_options[:proxyOffline] = @offline end def get_recipe() extract_path = File.join(Kameleon.env.build_path, File.basename(@cache_path, ".*")) FileUtils.mkdir_p extract_path execute("tar","-xf #{@cache_path} -C #{extract_path} ./recipe ./metadata") Kameleon.ui.info("Getting cached recipe") recipe_header = YAML::load(File.read(File.join(extract_path,"metadata","header"))) recipe_file = File.join(extract_path,"recipe",recipe_header[:recipe_path]) return recipe_file end def execute(cmd,args, dir=nil, environment={}) command = [cmd ] + args.split(" ") process = ChildProcess.build(*command) process.cwd = dir unless dir.nil? environment.each do |key, value| process.environment[key] = value end process.start process.wait end def common_prefix(paths) return '' if paths.empty? return paths.first.split('/').slice(0...-1).join('/') if paths.length <= 1 arr = paths.sort first = arr.first.to_s.split('/') last = arr.last.to_s.split('/') i = 0 i += 1 while first[i] == last[i] && i <= first.length first.slice(0, i).join('/') end end end kameleon-builder-2.11.0/lib/kameleon/shell.rb0000644000004100000410000003173715005702337021054 0ustar www-datawww-datarequire 'kameleon/utils' require 'shellwords' module Kameleon class Shell ECHO_CMD = "echo" READ_CHUNK_SIZE = 1048576 EXIT_TIMEOUT = 60 attr :exit_status attr :process attr :shell_cmd def initialize(context) @cmd = context.cmd.chomp @interactive_cmd = context.interactive_cmd.chomp @context_name = context.name @local_workdir = context.local_workdir @shell_workdir = context.workdir @proxy = context.proxy @env_files = context.env_files @bash_scripts_dir = File.join("kameleon_scripts", @context_name) @bashrc_file = File.join(@bash_scripts_dir, "bash_rc") @bash_history_file = File.join(@bash_scripts_dir, "bash_history") @bash_env_file = File.join(@bash_scripts_dir, "bash_env") @bash_status_file = File.join(@bash_scripts_dir, "bash_status") if File::directory?(f = File.join(Kameleon.source_root, "contrib")) @contrib_dir = f elsif File::directory?(f = '/usr/share/kameleon/contrib') @contrib_dir = f else raise "Could not find contrib dir" end @default_bashrc_file = File.join(@contrib_dir, 'kameleon_bashrc.sh') @cmd_tpl = ERB.new(File.read(File.join(@contrib_dir, "kameleon_exec_cmd.sh"))) @cmd_wrapper_tpl = ERB.new(File.read(File.join(@contrib_dir, "kameleon_exec_cmd_wrapper.sh"))) if @shell_workdir @bash_scripts_dir = File.join(@shell_workdir, @bash_scripts_dir) @bashrc_file = File.join(@shell_workdir, @bashrc_file) @bash_history_file = File.join(@shell_workdir, @bash_history_file) @bash_env_file = File.join(@shell_workdir, @bash_env_file) @bash_status_file = File.join(@shell_workdir, @bash_status_file) end ## Changing the default bashrc if the cache is activated @cache = Kameleon::Persistent_cache.instance @shell_cmd = "source #{@default_bashrc_file} 2> /dev/null; "\ "#{@cmd} --rcfile #{@bashrc_file}" @interactive_shell_cmd = "source #{@default_bashrc_file} 2> /dev/null; "\ "#{@interactive_cmd} --rcfile #{@bashrc_file}" Kameleon.ui.debug("Initialize shell (#{self})") # Injecting all variables of the options and assign the variables instance_variables.each do |v| Kameleon.ui.debug(" #{v} = #{instance_variable_get(v)}") end end def start @sent_first_cmd = false @process, @stdout, @stderr = fork("pipe") end def stop unless @process.nil? @process.stop(0) @process = nil end end def started? return !@process.nil? end def exited? return !@process.nil? && @process.exited? end def restart stop start end def send_file(source_path, remote_dest_path, chunk_size=READ_CHUNK_SIZE) copy_process, = fork("pipe") copy_process.io.stdin << init_shell_cmd copy_process.io.stdin << "cat > #{remote_dest_path}\n" copy_process.io.stdin.flush open(source_path, "rb") do |f| begin copy_process.io.stdin << f.read(chunk_size) end until f.eof? end copy_process.io.stdin.flush copy_process.io.stdin.close copy_process.wait copy_process.poll_for_exit(EXIT_TIMEOUT) end def init_shell_cmd ## log shell error message error = read_io(@stderr) init_stdout = read_io(@stdout) Kameleon.ui.error(error) unless error.empty? Kameleon.ui.info(init_stdout) unless init_stdout.empty? bashrc_content = "" if File.file?(@default_bashrc_file) tpl = ERB.new(File.read(@default_bashrc_file)) bashrc_content = tpl.result(binding) if @proxy != "" then tpl = ERB.new(File.read(File.join(Kameleon.source_root, "contrib", "proxy_env.sh"))) bashrc_content << "\n" + tpl.result(binding) end end # Inject sigint handler bashrc_content << <<-SCRIPT function save_state_handler { echo "Saved ENV in #{@bash_env_file} file" (comm -3 <(declare | sort) <(declare -f | sort)) > #{@bash_env_file} } trap save_state_handler EXIT SCRIPT bashrc = Shellwords.escape(bashrc_content) if @shell_workdir unless @shell_workdir.eql? "/" change_dir_cmd = "mkdir -p #{@shell_workdir} &&" end change_dir_cmd = "#{change_dir_cmd} cd #{@shell_workdir}" change_dir_cmd = Shellwords.escape(change_dir_cmd) end shell_cmd = "mkdir -p $(dirname #{@bashrc_file})\n" shell_cmd << "rm $(dirname #{@bashrc_file})/*\n" shell_cmd << "echo #{bashrc} > #{@bashrc_file}\n" unless change_dir_cmd.nil? shell_cmd << "echo #{change_dir_cmd} >> #{@bashrc_file}\n" end shell_cmd << "export KAMELEON_WORKDIR=$PWD\n" @env_files.each do |env_file| resolved_erb = "\n" + ERB.new(File.read(env_file)).result(binding) env_content = <<-SCRIPT #==================================================== # Begin content of user script '#{env_file.basename}' #==================================================== #{resolved_erb} #==================================================== # End content of user script '#{env_file.basename}' #==================================================== SCRIPT env_escaped_content = Shellwords.escape(env_content) shell_cmd << "echo #{env_escaped_content} >> #{@bashrc_file}\n" end shell_cmd << "source #{@bashrc_file}\n" shell_cmd << "(comm -3 <(declare | sort) <(declare -f | sort)) > #{@bash_env_file}\n" shell_cmd end def send_command cmd shell_cmd = "" unless @sent_first_cmd shell_cmd << init_shell_cmd @sent_first_cmd = true end cmd_content = Shellwords.escape(@cmd_tpl.result(binding)) cmd_wrapper_content = Shellwords.escape(@cmd_wrapper_tpl.result(binding)) shell_cmd << "mkdir -p $(dirname #{File.join(@bash_scripts_dir, "#{cmd}.sh" )})\n" shell_cmd << "echo #{cmd_content} > #{File.join(@bash_scripts_dir, "#{cmd}.sh" )}\n" shell_cmd << "echo #{cmd_wrapper_content} > #{File.join(@bash_scripts_dir, "#{cmd}_wrapper.sh" )}\n" shell_cmd << "bash #{File.join(@bash_scripts_dir, "#{cmd}_wrapper.sh" )}\n" @process.io.stdin.puts shell_cmd @process.io.stdin.flush end def execute(cmd, kwargs = {}) cmd_obj = Command.new(cmd) send_command(cmd_obj) iodata = {:stderr => { :io => @stderr, :name => 'stderr', :begin => false, :end => false, :begin_pat => cmd_obj.begin_err_pat, :end_pat => cmd_obj.end_err_pat, :redirect => kwargs[:stderr], :yield => lambda{|buf| yield(nil, buf)} }, :stdout => { :io => @stdout, :name => 'stdout', :begin => false, :end => false, :begin_pat => cmd_obj.begin_out_pat, :end_pat => cmd_obj.end_out_pat, :redirect => kwargs[:stdout], :yield => lambda{|buf| yield(buf, nil)} } } while true if @process.exited? raise ShellExited, "Process '#{@cmd}' exited..." end iodata.each do |_, iodat| if iodat[:end] and not iodat[:begin] if @process.exited? raise ShellExited, "Process '#{@cmd}' exited..." end raise ShellError, "Cannot read #{iodat[:begin]} from shell" end end if iodata.all? { |k, iodat| iodat[:end] and iodat[:begin]} break end readers = (iodata.map { |_, v| v[:io] unless v[:end] }) ready = IO.select(readers.compact, nil, nil, 0.1) ready ||= [[]] readers = ready[0] # Check the readers to see if they're ready if readers && !readers.empty? readers.each do |r| # Read from the IO object iodat = r == @stdout ? iodata[:stdout] : iodata[:stderr] data = read_io(r) # We don't need to do anything if the data is empty next if data.empty? if !iodat[:begin] && (m = iodat[:begin_pat].match(data)) iodat[:begin] = true data = m[1] end next unless iodat[:begin] and not iodat[:end] # ignore chaff if !iodat[:end] && (m = iodat[:end_pat].match(data)) iodat[:end] = true data = m[1] end next if data.empty? if iodat[:redirect] iodat[:redirect] << data else iodat[:yield].call data if block_given? end end end end if @process.exited? raise ShellExited, "Process '#{@cmd}' exited..." end iodata = nil return get_status end def start_interactive process, _, _ = fork("interactive") process.wait() end protected def get_status var_name = "__exit_status__" @process.io.stdin << "#{ ECHO_CMD } \"#{ var_name }=$(cat #{@bash_status_file})\"\n" @process.io.stdin.flush while((line = @stdout.gets)) if (m = %r/#{ var_name }\s*=\s*(.*)/.match line) exit_status = m[1] unless exit_status =~ /^\s*\d+\s*$/o raise ShellError, "could not determine exit status from <#{ exit_status.inspect }>" end @exit_status = Integer exit_status return @exit_status end end end def read_io(io) data = "" while true begin # Do a simple non-blocking read on the IO object data << io.read_nonblock(READ_CHUNK_SIZE) rescue Exception => e breakable = false if e.is_a?(EOFError) # An `EOFError` means this IO object is done! breakable = true elsif defined?(IO::WaitReadable) && e.is_a?(IO::WaitReadable) breakable = true elsif e.is_a?(Errno::EAGAIN) breakable = true end break if breakable raise end end data end def fork(io) if io.eql? "interactive" command = ["bash", "-c", @interactive_shell_cmd] Kameleon.ui.verbose("Starting interactive command: #{@interactive_shell_cmd.inspect}") else command = ["bash", "-c", @shell_cmd] Kameleon.ui.verbose("Starting command: #{@cmd.inspect}") end Kameleon.ui.debug("Starting shell process: #{ command.inspect}") ChildProcess.posix_spawn = true process = ChildProcess.build(*command) # Create the pipes so we can read the output in real time as # we execute the command. if io.eql? "pipe" stdout, stdout_writer = IO.pipe stderr, stderr_writer = IO.pipe process.io.stdout = stdout_writer process.io.stderr = stderr_writer # sets up pipe so process.io.stdin will be available after .start process.duplex = true elsif io.eql? "interactive" process.io.inherit! end # Start the process begin process.cwd = @local_workdir process.start # Wait to child starting sleep(0.2) rescue ChildProcess::LaunchError => e # Raise our own version of the error raise ShellError, "Cannot launch #{command.inspect}: #{e.message}" end if io.eql? "pipe" # Make sure the stdin does not buffer process.io.stdin.sync = true stdout_writer.close() stderr_writer.close() return process, stdout, stderr else return process, $stdout, $stderr end end class Command class << self def counter; @counter ||= 0; end def counter= n; @counter = n; end end attr :value attr :number attr :id attr :slug attr :begin_out attr :begin_out_pat attr :end_out attr :end_out_pat attr :begin_err attr :begin_err_pat attr :end_err attr :end_err_pat def initialize(raw) @value = raw.to_s.strip @number = self.class.counter @slug = Kameleon::Utils.generate_slug(@value)[0...30] @id = "%d_%d_%d" % [@number, $$, rand(Time.now.usec)] @begin_out = "__CMD_OUT_%s_BEGIN__" % @id @end_out = "__CMD_OUT_%s_END__" % @id @begin_out_pat = %r/#{ Regexp.escape(@begin_out) }(.*)/m @end_out_pat = %r/(.*)#{ Regexp.escape(@end_out) }/m @begin_err = "__CMD_ERR_%s_BEGIN__" % @id @end_err = "__CMD_ERR_%s_END__" % @id @begin_err_pat = %r/#{ Regexp.escape(@begin_err) }(.*)/m @end_err_pat = %r/(.*)#{ Regexp.escape(@end_err) }/m self.class.counter += 1 end def to_s "%05d_#{@slug}" % @number end end end end kameleon-builder-2.11.0/lib/kameleon/error.rb0000644000004100000410000000200215005702337021055 0ustar www-datawww-data module Kameleon class Error < ::StandardError attr_accessor :object def initialize(message=nil, object=nil) super(message) self.object = object end def self.status_code(code) define_method(:status_code) { code } end end class Exit < Error; status_code(0) ; end class KameleonError < Error; status_code(1) ; end class ExecError < Error; status_code(2) ; end class InternalError < Error; status_code(3) ; end class ContextError < Error; status_code(4) ; end class ContextClosed < Error; status_code(4) ; end class ShellError < Error; status_code(5) ; end class ShellExited < Error; status_code(5) ; end class RecipeError < Error; status_code(6) ; end class BuildError < Error; status_code(7) ; end class AbortError < Error; status_code(8) ; end class TemplateNotFound < Error; status_code(9) ; end class CacheError < Error; status_code(10) ; end class ExportError < Error; status_code(11) ; end class RepositoryError < Error; status_code(12) ; end end kameleon-builder-2.11.0/lib/kameleon/engine.rb0000644000004100000410000006605015005702337021206 0ustar www-datawww-datarequire 'kameleon/recipe' require 'kameleon/context' require 'kameleon/persistent_cache' require 'graphviz' module Kameleon class Engine attr_accessor :recipe attr_accessor :cwd def initialize(recipe, options) @options = options @recipe = recipe @cleaned_sections = [] @cwd = @recipe.global["kameleon_cwd"] @build_recipe_path = File.join(@cwd, ".build_recipe") @recipe.global["checkpointing_enabled"] = @options[:enable_checkpointing] ? "true" : "false" @recipe.global["persistent_cache"] = @options[:enable_cache] ? "true" : "false" build_recipe = load_build_recipe # restore previous build uuid unless build_recipe.nil? %w(kameleon_uuid kameleon_short_uuid).each do |key| @recipe.global[key] = build_recipe["global"][key] end end @checkpointing = @options[:enable_checkpointing] # Check if the recipe have checkpoint entry if @checkpointing && @recipe.checkpoint.nil? fail BuildError, "Checkpoint is unavailable for this recipe" end if @options[:enable_cache] || @options[:from_cache] then if @recipe.global["in_context"]["proxy_cache"].nil? then raise BuildError, "Missing variable for in context 'proxy_cache' when using the option --cache" end if @recipe.global["out_context"]["proxy_cache"].nil? then raise BuildError, "Missing variable for out context 'proxy_cache' when using the option --cache" end @cache = Kameleon::Persistent_cache.instance @cache.cwd = @cwd @cache.polipo_path = @options[:polipo_path] @cache.name = @recipe.name @cache.mode = @options[:enable_cache] ? :build : :from @cache.offline = @options[:proxy_offline] @cache.cache_path = @options[:from_cache] @cache.recipe_path = @recipe.path @cache.archive_format = @options[:cache_archive_compression] if @options[:proxy] != "" @cache.polipo_cmd_options['parentProxy'] = @options[:proxy] end if @options[:proxy_credentials] != "" @cache.polipo_cmd_options['parentAuthCredentials'] = @options[:proxy_credentials] end @recipe.global["proxy_local"] = "127.0.0.1:#{@cache.polipo_port}" @recipe.global["proxy_out"] = "#{@recipe.global['out_context']['proxy_cache']}:#{@cache.polipo_port}" @recipe.global["proxy_in"] = "#{@recipe.global['in_context']['proxy_cache']}:#{@cache.polipo_port}" elsif @options[:proxy] != "" if @options[:proxy_credentials] != "" proxy_url = "#{@options[:proxy_credentials]}@#{@options[:proxy]}" else proxy_url = "#{@options[:proxy]}" end @recipe.global["proxy_local"] = @recipe.global["proxy_out"] = @recipe.global["proxy_in"] = proxy_url end @recipe.resolve! if @options[:enable_cache] || @options[:from_cache] then @cache.recipe_files = @recipe.all_files end unless @options[:no_create_build_dir] begin Kameleon.ui.info("Creating kameleon build directory: #{@cwd}") FileUtils.mkdir_p @cwd rescue raise BuildError, "Failed to create build directory #{@cwd}" end @cache.start if @cache build_contexts end end def build_contexts lazyload = @options.fetch(:lazyload, true) fail_silently = @options.fetch(:fail_silently, false) Kameleon.ui.debug("Building local context [local]") @local_context = Context.new("local", "bash", "bash", @cwd, "", @cwd, @recipe.env_files, :proxy => @recipe.global["proxy_local"], :lazyload => lazyload, :fail_silently => false) Kameleon.ui.debug("Building external context [out]") @out_context = Context.new("out", @recipe.global["out_context"]["cmd"], @recipe.global["out_context"]["interactive_cmd"], @recipe.global["out_context"]["workdir"], @recipe.global["out_context"]["exec_prefix"], @cwd, @recipe.env_files, :proxy => @recipe.global["proxy_out"], :lazyload => lazyload, :fail_silently => fail_silently) Kameleon.ui.debug("Building internal context [in]") @in_context = Context.new("in", @recipe.global["in_context"]["cmd"], @recipe.global["in_context"]["interactive_cmd"], @recipe.global["in_context"]["workdir"], @recipe.global["in_context"]["exec_prefix"], @cwd, @recipe.env_files, :proxy => @recipe.global["proxy_in"], :lazyload => lazyload, :fail_silently => fail_silently) end def reload_contexts [@local_context, @out_context, @in_context].each do |ctx| ctx.reload if ctx.shell.started? end end def saving_steps_files @recipe.files.each do |file| Kameleon.ui.info("File #{file} loaded from the recipe") end end def create_cache_directory(step_name) Kameleon.ui.debug("Creating directory for cache #{step_name}") directory_name = @cache.cache_dir + "/#{step_name}" FileUtils.mkdir_p directory_name directory_name end def create_checkpoint(microstep_id) @recipe.checkpoint["create"].each do |cmd| safe_exec_cmd(cmd.dup.gsub!("@microstep_id", microstep_id), :log_level => "warn") end end def checkpoint_enabled? @recipe.checkpoint["enabled?"].each do |cmd| exec_cmd(cmd, :log_level => "debug") end return true rescue ExecError return false end def apply_checkpoint(microstep_id) @recipe.checkpoint["apply"].each do |cmd| safe_exec_cmd(cmd.dup.gsub!("@microstep_id", microstep_id)) end end def list_checkpoints if @list_checkpoints.nil? # get existing checkpoints on the system existing_checkpoint_str = "" @recipe.checkpoint["list"].each do |cmd| safe_exec_cmd(cmd, :stdout => existing_checkpoint_str) end existing_checkpoint_ids = existing_checkpoint_str.split(/\r?\n/) # get sorted checkpoints by microsteps order @list_checkpoints = [] @recipe.all_checkpoints.each do |checkpoint| @list_checkpoints.push(checkpoint) if existing_checkpoint_ids.include?(checkpoint["id"]) end end return @list_checkpoints end def do_steps(section_name) section = @recipe.sections.fetch(section_name) section.sequence do |macrostep| checkpointed = false macrostep_time = Time.now.to_i macrostep_checkpoint_duration = 0 if @cache then Kameleon.ui.debug("Starting proxy cache server for macrostep '#{macrostep.name}'...") # the following function start a polipo web proxy and stops a previous run dir_cache = @cache.create_cache_directory(macrostep.name) unless @cache.start_web_proxy_in(dir_cache) raise CacheError, "The cache process fail to start" end end macrostep.sequence do |microstep| microstep_time = Time.now.to_i microstep_checkpoint_duration = 0 step_prefix = "Step #{ microstep.order }: " Kameleon.ui.info("#{step_prefix}#{ microstep.slug }") if @checkpointing if microstep.on_checkpoint == "skip" Kameleon.ui.msg("--> Skip microstep as requested when checkpointing is activated") next end if microstep.has_checkpoint_ahead and microstep.on_checkpoint != "redo" Kameleon.ui.msg("--> Checkpoint ahead, do nothing") else begin Kameleon.ui.msg("--> Running the step...") microstep.commands.each do |cmd| safe_exec_cmd(cmd) end rescue SystemExit, Interrupt reload_contexts breakpoint(nil) end if checkpoint_enabled? if (@options[:microstep_checkpoints].downcase == "first" and checkpointed) Kameleon.ui.msg("--> Do not create a checkpoint for this microstep: macrostep already checkpointed once") elsif microstep.on_checkpoint == "redo" Kameleon.ui.msg("--> Do not create a checkpoint for this microstep: always redo microstep") elsif microstep.on_checkpoint == "disabled" Kameleon.ui.msg("--> Do not create a checkpoint for this microstep: disabled in the microstep definition") elsif not microstep.in_checkpoint_window if @end_checkpoint.nil? unless @begin_checkpoint.nil? msg = "only after step '#{@begin_checkpoint['step']}'" end else if @begin_checkpoint.nil? msg = "not after step '#{@end_checkpoint['step']}'" else msg = "only between steps '#{@begin_checkpoint['step']}' and '#{@end_checkpoint['step']}'" end end Kameleon.ui.msg("--> Do not create a checkpoint for this microstep: #{msg}") else microstep_checkpoint_time = Time.now.to_i Kameleon.ui.msg("--> Creating checkpoint: '#{@recipe.all_checkpoints.select{|c| c['id'] == microstep.identifier}.first['step']}' (#{microstep.identifier})") create_checkpoint(microstep.identifier) checkpointed = true microstep_checkpoint_duration = Time.now.to_i - microstep_checkpoint_time macrostep_checkpoint_duration += microstep_checkpoint_duration Kameleon.ui.verbose("Checkpoint creation for MicroStep #{microstep.name} took: #{microstep_checkpoint_duration} secs") end else Kameleon.ui.msg("--> Do not create a checkpoint for this microstep: disabled in backend") end end else if microstep.on_checkpoint == "only" Kameleon.ui.msg("--> Skip microstep as requested when checkpointing is not activated") next else begin Kameleon.ui.msg("--> Running the step...") microstep.commands.each do |cmd| safe_exec_cmd(cmd) end rescue SystemExit, Interrupt reload_contexts breakpoint(nil) end end end Kameleon.ui.verbose("MicroStep #{microstep.name} took: #{Time.now.to_i - microstep_time - microstep_checkpoint_duration} secs") end Kameleon.ui.info("Step #{macrostep.name} took: #{Time.now.to_i - macrostep_time - macrostep_checkpoint_duration} secs") end @cleaned_sections.push(section.name) end def safe_exec_cmd(cmd, kwargs = {}) finished = false begin exec_cmd(cmd, kwargs) finished = true rescue ContextClosed => e Kameleon.ui.warn("#{e.message}") finished = true rescue SystemExit, Interrupt, ExecError reload_contexts finished = rescue_exec_error(cmd) end until finished end def exec_cmd(cmd, kwargs = {}) map = {"exec_in" => @in_context, "exec_out" => @out_context, "exec_local" => @local_context} case cmd.key when "breakpoint" breakpoint(cmd.value) when "reload_context" context = "exec_" + cmd.value expected_names = map.keys.map { |k| k.gsub "exec_", "" } unless map.keys.include? context Kameleon.ui.error("Invalid context name arguments. Expected: "\ "#{expected_names}") fail ExecError else map[context].reload end when "exec_in", "exec_out", "exec_local" if kwargs[:only_with_context] and map[cmd.key].closed? Kameleon.ui.debug("Not executing #{cmd.key} command (context closed): #{cmd.value}") else map[cmd.key].execute(cmd.value, kwargs) end when "pipe" first_cmd, second_cmd = cmd.value expected_cmds = ["exec_in", "exec_out", "exec_local"] execute = true [first_cmd.key, second_cmd.key].each do |key| unless expected_cmds.include?(key) Kameleon.ui.error("Invalid pipe arguments. Expected: "\ "#{expected_cmds}") fail ExecError end if kwargs[:only_with_context] and map[key].closed? Kameleon.ui.debug("Not executing pipe command (context closed sub command #{key})") execute = false end end if execute first_context = map[first_cmd.key] second_context = map[second_cmd.key] @cache.cache_cmd_raw(cmd.raw_cmd_id) if @cache first_context.pipe(first_cmd.value, second_cmd.value, second_context, kwargs) end when "rescue" unless cmd.value.length == 2 Kameleon.ui.error("Invalid 'rescue' command arguments. Expecting 2 sub commands") fail ExecError end first_cmd, second_cmd = cmd.value begin exec_cmd(first_cmd, kwargs) rescue ExecError safe_exec_cmd(second_cmd, kwargs) end when "test" unless cmd.value.length == 2 or cmd.value.length == 3 Kameleon.ui.error("Invalid 'test' command arguments. Expecting 2 or 3 sub commands") fail ExecError end first_cmd, second_cmd, third_cmd = cmd.value begin Kameleon.ui.debug("Execute test condition") # Drop any :only_with_context flag, so that "if" fails if a closed # context exception occurs. In that case, the "else" statement must # be executed rather than the "then" statement. exec_cmd(first_cmd, kwargs.reject {|k| k == :only_with_context}) rescue ExecError unless third_cmd.nil? Kameleon.ui.debug("Execute test 'else' statment'") exec_cmd(third_cmd, kwargs) end else Kameleon.ui.debug("Execute test 'then' statment'") exec_cmd(second_cmd, kwargs) end when "group" cmds = cmd.value cmds.each do |cmd| exec_cmd(cmd, kwargs) end else Kameleon.ui.warn("Unknown command: #{cmd.key}") end end def breakpoint(message, kwargs = {}) message = "Kameleon breakpoint!" if message.nil? message.split( /\r?\n/ ).each {|m| Kameleon.ui.error "#{m}" } enable_retry = kwargs[:enable_retry] msg = "" msg << "Press [r] to retry\n" if enable_retry msg << "Press [c] to continue with execution" msg << "\nPress [a] to abort execution" msg << "\nPress [l] to switch to local_context shell" msg << "\nPress [o] to switch to out_context shell" msg << "\nPress [i] to switch to in_context shell" responses = {"c" => "continue", "a" => "abort"} responses["r"] = "retry" if enable_retry responses.merge!({"l" => "launch local_context"}) responses.merge!({"o" => "launch out_context"}) responses.merge!({"i" => "launch in_context"}) while true msg.split( /\r?\n/ ).each {|m| Kameleon.ui.info "#{m}" } if Kameleon.env.script? answer = "a" else answer = Kameleon.ui.ask "answer ? [" + responses.keys().join("/") + "]: " end raise AbortError, "Execution aborted..." if answer.nil? answer.chomp! if responses.keys.include?(answer) Kameleon.ui.info("User choice: [#{answer}] #{responses[answer]}") if ["o", "i", "l"].include?(answer) if answer.eql? "l" @local_context.start_shell elsif answer.eql? "o" @out_context.start_shell else @in_context.start_shell end Kameleon.ui.info("Getting back to Kameleon...") elsif answer.eql? "a" raise AbortError, "Execution aborted..." elsif answer.eql? "c" ## resetting the exit status @in_context.execute("true") unless @in_context.closed? @out_context.execute("true") unless @out_context.closed? @local_context.execute("true") unless @local_context.closed? return true elsif answer.eql? "r" Kameleon.ui.info("Retrying the previous command...") return false end end end end def rescue_exec_error(cmd) message = "Error occured when executing the following command:\n" cmd.string_cmd.split( /\r?\n/ ).each {|m| message << "\n> #{m}" } if Kameleon.env.script? raise ExecError, message end return breakpoint(message, :enable_retry => true) end def clean(kwargs = {}) kwargs = kwargs.merge({:only_with_context => true}) if kwargs.fetch(:with_checkpoint, false) Kameleon.ui.info("Removing all checkpoints") @recipe.checkpoint["clear"].each do |cmd| begin exec_cmd(cmd, kwargs) rescue Kameleon.ui.warn("An error occurred while executing: #{cmd.value}") end end end @recipe.sections.values.each do |section| next if @cleaned_sections.include?(section.name) Kameleon.ui.info("Cleaning #{section.name} section") section.clean_macrostep.sequence do |microstep| if @checkpointing if microstep.on_checkpoint == "skip" next end else if microstep.on_checkpoint == "only" next end end microstep.commands.each do |cmd| begin exec_cmd(cmd, kwargs) rescue Kameleon.ui.warn("An error occurred while executing: #{cmd.value}") end end end end @cache.stop_web_proxy if @options[:enable_cache] ## stopping polipo end def dryrun def relative_or_absolute_path(path) if @options[:relative] return path.relative_path_from(Pathname(Dir.pwd)) else return path end end if @options[:list_checkpoints] if @recipe.all_checkpoints.empty? Kameleon.ui.shell.say "No checkpoints would be created by recipe '#{recipe.name}':" else Kameleon.ui.shell.say "The following checkpoints would be created by recipe '#{recipe.name}':" tp @recipe.all_checkpoints, {"id" => {:width => 20}}, { "step" => {:width => 60}} end else Kameleon.ui.shell.say "" Kameleon.ui.shell.say "#{ @recipe.name } ", :bold Kameleon.ui.shell.say "(#{ relative_or_absolute_path(@recipe.path) })", :cyan ["bootstrap", "setup", "export"].each do |section_name| section = @recipe.sections.fetch(section_name) Kameleon.ui.shell.say "[" << section.name.capitalize << "]", :red section.sequence do |macrostep| Kameleon.ui.shell.say " " Kameleon.ui.shell.say "#{macrostep.name} ", :bold if macrostep.path Kameleon.ui.shell.say "(#{ relative_or_absolute_path(macrostep.path) })", :cyan else Kameleon.ui.shell.say "(internal)", :cyan end macrostep.sequence do |microstep| Kameleon.ui.shell.say " --> ", :magenta Kameleon.ui.shell.say "#{ microstep.order } ", :green Kameleon.ui.shell.say "#{ microstep.name }", :yellow end end end Kameleon.ui.shell.say "" end end def dag(graph, color, recipes_only) if graph.nil? graph = GraphViz::new( "G" ) end recipe_path = @recipe.path.relative_path_from(Pathname(Dir.pwd)).to_s colorscheme = "set18" color = (color % 8 + 1).to_s g_recipes = graph.add_graph( "cluster R:recipes" ) g_recipes['label'] = 'Recipes' g_recipes['style'] = 'dashed' n_recipe = g_recipes.add_nodes(recipe_path) n_recipe['label'] = recipe_path n_recipe['shape'] = 'Mdiamond' n_recipe['colorscheme'] = colorscheme n_recipe['color'] = color (@recipe.base_recipes_files - [@recipe.path]).uniq.each do |base_recipe_path| Kameleon.ui.debug("Dag add node #{base_recipe_path}") n_base_recipe = g_recipes.add_nodes(base_recipe_path.relative_path_from(Pathname(Dir.pwd)).to_s) n_base_recipe['shape'] = 'Mdiamond' edge = graph.add_edges(n_base_recipe, n_recipe) if base_recipe_path == @recipe.extended_recipe_file Kameleon.ui.debug("This is the extended recipe") edge['colorscheme'] = colorscheme edge['color'] = color else edge['style'] = 'dashed' end end n_prev = n_recipe if recipes_only Kameleon.ui.debug("As requested, only show recipes") else ["bootstrap", "setup", "export"].each do |section_name| Kameleon.ui.debug("Dag add section #{section_name}") section = @recipe.sections.fetch(section_name) g_section = graph.add_graph( "cluster S:#{ section_name }" ) g_section['label'] = section_name.capitalize section.sequence do |macrostep| Kameleon.ui.debug("Dag add macrostep #{macrostep.name}") if macrostep.path.nil? macrostep_name = macrostep.name else macrostep_name = macrostep.path.relative_path_from(Pathname(Dir.pwd)).to_s macrostep_name.chomp!(".yaml") macrostep_name.sub!(/^steps\//, "") end g_macrostep = g_section.add_graph( "cluster M:#{ macrostep_name }") g_macrostep['label'] = macrostep_name g_macrostep['style'] = 'filled' g_macrostep['color'] = 'gray' macrostep.sequence do |microstep| Kameleon.ui.debug("Dag add microstep #{microstep.name}") n_microstep = g_macrostep.add_nodes("m:#{macrostep_name}/#{microstep.name}") n_microstep['label'] = microstep.name n_microstep['style'] = 'filled' n_microstep['color'] = 'white' edge = graph.add_edges(n_prev, n_microstep) edge['colorscheme'] = colorscheme edge['color'] = color n_prev = n_microstep end end end n_end = graph.add_nodes('end') n_end['label'] = 'END' n_end['shape'] = 'Msquare' edge = graph.add_edges(n_prev, n_end) edge['colorscheme'] = colorscheme edge['color'] = color end Kameleon.ui.info "-> Draw DAG for #{recipe_path}" return graph end def build if @checkpointing if @options[:from_checkpoint].nil? || @options[:from_checkpoint] == "last" @from_checkpoint = list_checkpoints.last else @from_checkpoint = list_checkpoints.select {|c| c["id"] == @options[:from_checkpoint] || c["step"] == @options[:from_checkpoint] }.last if @from_checkpoint.nil? fail BuildError, "Unknown checkpoint '#{@options[:from_checkpoint]}'." \ " You may use the list checkpoints option to find a valid checkpoint." end end unless @from_checkpoint.nil? # no checkpoint available at all Kameleon.ui.info("Restoring last build from step: #{@from_checkpoint["step"]}") apply_checkpoint @from_checkpoint["id"] @recipe.microsteps.each do |microstep| microstep.has_checkpoint_ahead = true if microstep.identifier == @from_checkpoint["id"] break end end end @begin_checkpoint = nil unless @options[:begin_checkpoint].nil? @begin_checkpoint = @recipe.all_checkpoints.select {|c| c["id"] == @options[:begin_checkpoint] || c["step"] == @options[:begin_checkpoint] }.first if @begin_checkpoint.nil? fail BuildError, "Unknown checkpoint '#{@options[:begin_checkpoint]}'." \ " You may use the dryrun and list checkpoints options to find a valid checkpoint." end end @end_checkpoint = nil unless @options[:end_checkpoint].nil? @end_checkpoint = @recipe.all_checkpoints.select {|c| c["id"] == @options[:end_checkpoint] || c["step"] == @options[:end_checkpoint] }.last if @end_checkpoint.nil? fail BuildError, "Unknown checkpoint '#{@options[:end_checkpoint]}'." \ " You may use the dryrun and list checkpoints options to find a valid checkpoint." end end do_checkpoint = @begin_checkpoint.nil? @recipe.microsteps.each do |microstep| if not do_checkpoint and not @begin_checkpoint.nil? and @begin_checkpoint["id"] == microstep.identifier do_checkpoint = true end microstep.in_checkpoint_window = do_checkpoint if do_checkpoint and not @end_checkpoint.nil? and @end_checkpoint["id"] == microstep.identifier do_checkpoint = false end end end dump_build_recipe begin ["bootstrap", "setup", "export"].each do |section| do_steps(section) end @cache.stop if @cache clean rescue Exception => e if e.is_a?(AbortError) Kameleon.ui.error("Aborted...") elsif e.is_a?(SystemExit) || e.is_a?(Interrupt) Kameleon.ui.error("Interrupted...") @out_context.reload if @out_context.already_loaded? @in_context.reload if @in_context.already_loaded? @local_context.reload if @local_context.already_loaded? else Kameleon.ui.error("fatal error...") end Kameleon.ui.warn("Waiting for cleanup before exiting...") clean @out_context.close! @in_context.close! @local_context.close! raise e end end def dump_build_recipe File.open(@build_recipe_path, 'w') do |f| f.write @recipe.to_hash.to_yaml end end def load_build_recipe if File.file?(@build_recipe_path) result = YAML.unsafe_load_file(@build_recipe_path) return result if result end return nil end def pretty_checkpoints_list list_checkpoints if list_checkpoints.empty? Kameleon.ui.shell.say "No checkpoint found for recipe '#{recipe.name}'" else Kameleon.ui.shell.say "The following checkpoints are available for recipe '#{recipe.name}':" tp list_checkpoints, {"id" => {:width => 20}}, { "step" => {:width => 60}} end end end end kameleon-builder-2.11.0/lib/kameleon/compat.rb0000644000004100000410000000220215005702337021211 0ustar www-datawww-datarequire 'securerandom' if RUBY_VERSION < "1.9.3" # Backport of missing SecureRandom methods from 1.9 # Snippet from http://softover.com/UUID_in_Ruby_1.8 module SecureRandom class << self def method_missing(method_sym, *arguments, &block) case method_sym when :urlsafe_base64 r19_urlsafe_base64(*arguments) when :uuid r19_uuid(*arguments) else super end end private def r19_urlsafe_base64(n=nil, padding=false) s = [random_bytes(n)].pack("m*") s.delete!("\n") s.tr!("+/", "-_") s.delete!("=") if !padding s end def r19_uuid ary = random_bytes(16).unpack("NnnnnN") ary[2] = (ary[2] & 0x0fff) | 0x4000 ary[3] = (ary[3] & 0x3fff) | 0x8000 "%08x-%04x-%04x-%04x-%04x%08x" % ary end end end end class Object ## # @person ? @person.name :nil # vs # @person.try(:name) def try(method) send method if respond_to? method end end class Hash def self.try_convert(obj) obj.try(:to_hash) end def flatten to_a.flatten! end end kameleon-builder-2.11.0/lib/kameleon/step.rb0000644000004100000410000001551115005702337020710 0ustar www-datawww-datarequire 'digest' module Kameleon class Command attr_accessor :string_cmd attr_accessor :raw_cmd_id attr_accessor :microstep_name attr_accessor :identifier def initialize(yaml_cmd, microstep_name) @string_cmd = YAML.dump(yaml_cmd).gsub("---", "").strip @raw_cmd_id = Digest::SHA1.hexdigest(YAML.dump(yaml_cmd).gsub("---", "").strip) @microstep_name = microstep_name @identifier = nil end def resolve! key value end def key if @key.nil? object = YAML.unsafe_load(@string_cmd) if object.kind_of? String @key = object else @key = object.keys.first end end @key rescue lines = @string_cmd.split( /\r?\n/ ).map {|l| "> #{l}" } fail RecipeError, "Syntax error for microstep #{@microstep_name}: \n"\ "#{ lines.join "\n"}" end def value if @value.nil? Kameleon.ui.debug("Parsed string = #{@string_cmd}") object = YAML.unsafe_load(@string_cmd) if object.kind_of? Command @value = object elsif object.kind_of? String @value = nil else raise RecipeError unless object.kind_of? Hash raise RecipeError unless object.keys.count == 1 _, val = object.first unless val.kind_of?(Array) val = val.to_s end # Nested commands if val.kind_of? Array val = val.map { |item| Command.new(item, @microstep_name) } end @value = val end end @value rescue fail RecipeError, "Syntax error after variable resolution for microstep #{@microstep_name}, parsed string =\n"\ "#{@string_cmd}\n"\ "Maybe you should remove trailing newline from variable using '>-' or '|-'" end def to_array if value.kind_of? Array map = value.map { |val| val.to_array } return { key => map } else return { key => value } end end def string_cmd=(str) Kameleon.ui.debug("Set string_cmd to '#{str}' and clear cached value") @string_cmd = str @value = nil end def remaster_string_cmd_from_value! self.string_cmd = YAML.dump(to_array).gsub("---", "").strip return self end def gsub!(arg1, arg2) if value.kind_of? Array value.each { |cmd| cmd.gsub!(arg1, arg2) } else @value.gsub!(arg1, arg2) end remaster_string_cmd_from_value! return self end end class Microstep attr_accessor :commands attr_accessor :name attr_accessor :identifier attr_accessor :slug attr_accessor :has_checkpoint_ahead attr_accessor :in_checkpoint_window attr_accessor :on_checkpoint attr_accessor :order def initialize(string_or_hash) @identifier = nil @has_checkpoint_ahead = false @in_checkpoint_window = true @on_checkpoint = "use_cache" @commands = [] @name, cmd_list = string_or_hash.first cmd_list.each do |cmd_hash| if cmd_hash.kind_of? Command @commands.push cmd_hash else if cmd_hash.kind_of?(Hash) && cmd_hash.keys.first == "on_checkpoint" @on_checkpoint = cmd_hash["on_checkpoint"] else @commands.push Command.new(cmd_hash, @name) end end end rescue fail RecipeError, "Syntax error for microstep #{name}" end def resolve! @commands.each {|cmd| cmd.resolve! } end def gsub!(arg1, arg2) @commands.each {|cmd| cmd.gsub!(arg1, arg2) } end def unshift(cmd_list) cmd_list.reverse.each {|cmd| @commands.unshift cmd} end def push(cmd) @commands.push cmd end def calculate_identifier(salt) commands_str = @commands.map { |cmd| cmd.string_cmd.to_s } content_id = commands_str.join(' ') + salt @identifier = "#{ Digest::SHA1.hexdigest content_id }"[0..11] @commands.each do |cmd| map_id = cmd.string_cmd.to_s + @identifier cmd.identifier = "#{ Digest::SHA1.hexdigest map_id }"[0..11] end @identifier end def to_array microstep_array = @commands.map do |cmd| cmd.to_array end return microstep_array end end class Macrostep attr_accessor :name attr_accessor :clean_microsteps attr_accessor :init_microsteps attr_accessor :microsteps attr_accessor :path attr_accessor :variables def initialize(name, microsteps, variables, path) @name = name @variables = variables @path = path @microsteps = microsteps @clean_microsteps = [] @init_microsteps = [] end def resolve_variables!(global, recipe) # Resolve dynamically-defined variables !! tmp_resolved_vars = {} @variables.clone.each do |key, value| yaml_vars = { key => value }.to_yaml.chomp yaml_resolved = Utils.resolve_vars(yaml_vars, @path, tmp_resolved_vars.merge(global), recipe) tmp_resolved_vars.merge! YAML.unsafe_load(yaml_resolved.chomp) end @variables.merge! tmp_resolved_vars @microsteps.each do |m| m.commands.each do |cmd| cmd.string_cmd = Utils.resolve_vars(cmd.string_cmd, @path, global.merge(@variables), recipe) end end end def sequence @init_microsteps.each { |m| yield m } @microsteps.each { |m| yield m } @clean_microsteps.each { |m| yield m } end def to_array macrostep_array = [] @variables.each do |k, v| macrostep_array.push({ k => v }) end sequence do |microstep| macrostep_array.push({ microstep.name => microstep.to_array }) end return macrostep_array end end class Section attr_accessor :name attr_accessor :clean_macrostep attr_accessor :init_macrostep attr_accessor :macrosteps def initialize(name) @name = name @clean_macrostep = Macrostep.new("_clean_#{name}", [], {}, nil) @init_macrostep = Macrostep.new("_init_#{name}", [], {}, nil) @macrosteps = [] end def sequence yield @init_macrostep @macrosteps.each { |m| yield m } yield @clean_macrostep end def to_array section_array = [] sequence do |macrostep| macrostep.sequence do |microstep| hash = { "identifier" => microstep.identifier.to_s, "cmds" => microstep.to_array } section_array.push({ microstep.slug => hash }) end end return section_array end end end kameleon-builder-2.11.0/lib/kameleon/repository.rb0000644000004100000410000000476315005702337022163 0ustar www-datawww-datarequire 'kameleon/utils' require 'kameleon/step' module Kameleon class Repository def self.check_git_binary git_path ||= Utils.which("git") if git_path.nil? then raise KameleonError, "git binary not found, make sure it is in your current PATH" end end def self.add(name, url, kwargs = {}) check_git_binary cmd = ["git", "clone"] if kwargs[:branch] cmd.push("-b", kwargs[:branch]) end cmd.push("--", url, File.join(Kameleon.env.repositories_path, name)) process = ChildProcess.build(*cmd) process.io.inherit! process.start process.wait process.stop end def self.update(name) check_git_binary git_repo = File.join(Kameleon.env.repositories_path, name) raise RepositoryError, "Repository not found '#{name}'" if not File.directory?(git_repo) cmd = ["git", "--git-dir", File.join(git_repo, ".git"), "--work-tree", git_repo, "pull", "--ff-only"] process = ChildProcess.build(*cmd) process.io.inherit! process.start process.wait process.stop end def self.list(kwargs = {}) Dir["#{Kameleon.env.repositories_path}/*"].each do |repo_path| if kwargs[:git] show_git_repository(repo_path) else Kameleon.ui.info File.basename(repo_path) end end end def self.remove(name) repo_path = File.join(Kameleon.env.repositories_path, name) raise RepositoryError, "Repository not found '#{name}'" if not File.directory?(repo_path) Kameleon.ui.shell.say "Removing: ", :red, false show_git_repository(repo_path) FileUtils.rm_rf(repo_path) end def self.show_git_repository(repo_path) cmd = ["git", "remote", "-v"] r, w = IO.pipe process = ChildProcess.build(*cmd) process.io.stdout = w process.cwd = repo_path process.start w.close url = r.readline.split[1] process.wait process.stop cmd = ["git", "rev-parse", "--abbrev-ref", "HEAD"] r, w = IO.pipe process = ChildProcess.build(*cmd) process.io.stdout = w process.cwd = repo_path process.start w.close branch = r.readline.chomp process.wait process.stop Kameleon.ui.shell.say "#{File.basename("#{repo_path}")}", nil, false Kameleon.ui.shell.say " <-", :magenta, false Kameleon.ui.shell.say " #{url}", :cyan, false Kameleon.ui.shell.say " (#{branch})", :yellow end end end kameleon-builder-2.11.0/lib/kameleon/cli.rb0000644000004100000410000005246315005702337020513 0ustar www-datawww-datarequire 'kameleon/engine' require 'kameleon/recipe' require 'kameleon/utils' require 'tempfile' require 'graphviz' # ruby-graphviz gem (GraphViz class), not the graphviz gem (Graphviz module) module Kameleon module CLI class Repository < Thor include Thor::Actions desc "add ", "Add a new repository named cloned from " method_option :branch, :type => :string, :default => nil, :desc => "checkout ", :aliases => "-b" def add(name, url) Kameleon::Repository.add(name, url, options) end desc "list", "List available repositories" method_option :git, :type => :boolean, :default => true, :desc => "show the git repository and branch each repository comes from" def list Kameleon::Repository.list(options) end desc "update ", "Update repository named from git" def update(name) Kameleon::Repository.update(name) end desc "remove ", "Remove repository named " def remove(name) Kameleon::Repository.remove(name) end desc "commands", "List all available commands", :hide => true def commands puts Repository.all_commands.keys - ["commands"] end map %w(ls) => :list map %w(rm) => :remove map %w(completions) => :commands end class Template < Thor include Thor::Actions def self.source_root Kameleon.env.repositories_path end desc "list", "List all available templates" method_option :progress, :type => :boolean, :default => true, :desc => "Show progress bar while resolving templates", :aliases => "-p" method_option :filter, :type => :string, :default => nil, :desc => "Filter templates with the given regexp", :aliases => "-f" def list Kameleon.ui.shell.say "Recipe templates available in: ", :red, false Kameleon.ui.shell.say Kameleon.env.repositories_path.to_s, :yellow Utils.list_recipes(Kameleon.env.repositories_path, options[:filter], options[:progress], true) end desc "import ", "Import the given template" method_option :global, :type => :hash, :default => {}, :aliases => "-g", :desc => "Set custom global variables" def import(template_name) Kameleon.env.root_dir = Kameleon.env.repositories_path template_path = File.join(Kameleon.env.repositories_path, template_name) unless template_name.end_with? '.yaml' template_path = template_path + '.yaml' end # Manage global as it is not passed to env by default if options[:global] Kameleon.env.global.merge!(options[:global]) end begin tpl = RecipeTemplate.new(template_path) tpl.resolve! :strict => false rescue raise if Kameleon.ui.level("verbose") raise TemplateNotFound, "Template '#{template_name}' invalid (try" \ " --verbose) or not found. To see all templates, run the command "\ "`kameleon template list`" else tpl.all_files.each do |path| relative_path = path.relative_path_from(Kameleon.env.repositories_path) dst = File.join(Kameleon.env.workspace, relative_path) copy_file(path, dst) chmod(dst, File.stat(path).mode, {:verbose=>false}) end end end desc "info ", "Display detailed information about a template" method_option :global, :type => :hash, :default => {}, :aliases => "-g", :desc => "Set custom global variables" def info(template_name) Kameleon.env.root_dir = Kameleon.env.repositories_path template_path = File.join(Kameleon.env.repositories_path, template_name) unless template_name.end_with? '.yaml' template_path = template_path + '.yaml' end # Manage global as it is not passed to env by default if options[:global] Kameleon.env.global.merge!(options[:global]) end tpl = RecipeTemplate.new(template_path) tpl.resolve! :strict => false tpl.display_info(false) end desc "erb ", "Create a extend recipe ERB file" def erb(path) if File.directory?(path) erb_file = Pathname.new(path).join(Kameleon.default_values[:extend_yaml_erb]) elsif File.file?(path) and path.end_with?(".yaml") erb_file = Pathname.new(path.gsub(%r{^(.+?/)?([^/]+?)(\.yaml)?$},'\1.\2') + Kameleon.default_values[:extend_yaml_erb]) else fail KameleonError, "Invalid path '#{path}', please give a path to a yaml file or a directory" end Kameleon.ui.verbose("Create extend recipe ERB '#{erb_file}'") copy_file(Pathname.new(Kameleon.erb_dirpath).join("extend.yaml.erb"), erb_file) end desc "commands", "List all available commands", :hide => true def commands puts Template.all_commands.keys - ["commands"] end map %w(ls) => :list map %w(completions) => :commands end end class Main < Thor include Thor::Actions desc 'repository ', 'Manage repositories of recipes' subcommand 'repository', CLI::Repository desc 'template ', 'List and import templates' subcommand 'template', CLI::Template class_option :color, :type => :boolean, :default => Kameleon.default_values[:color], :desc => "Enable colorization in output" class_option :verbose, :type => :boolean, :default => Kameleon.default_values[:verbose], :desc => "Enable verbose output for kameleon users" class_option :debug, :type => :boolean, :default => Kameleon.default_values[:debug], :desc => "Enable debug output for kameleon developers" class_option :script, :type => :boolean, :default => Kameleon.default_values[:script], :desc => "Never prompt for user intervention", :aliases => "-s" desc "version", "Print the Kameleon's version information" def version puts "Kameleon version #{Kameleon::VERSION}" end def self.source_root Kameleon.env.repositories_path end desc "list", "List all defined recipes in the current directory" method_option :progress, :type => :boolean, :default => false, :desc => "Show progress bar while resolving recipes", :aliases => "-p" method_option :filter, :type => :string, :default => nil, :desc => "Filter recipes with the given regexp", :aliases => "-f" def list Kameleon.ui.shell.say "Workspace recipes:", :red Utils.list_recipes(Kameleon.env.workspace, options[:filter], options[:progress]) end desc "new ", "Create a new recipe from template " method_option :global, :type => :hash, :default => {}, :aliases => "-g", :desc => "Set custom global variables" def new(recipe_name, template_name) Kameleon.env.root_dir = Kameleon.env.repositories_path unless template_name.end_with? '.yaml' template_name = template_name + '.yaml' end unless recipe_name.end_with? '.yaml' recipe_name = recipe_name + '.yaml' end if recipe_name == template_name fail RecipeError, "Recipe path should be different from template name" end template_path = File.join(Kameleon.env.repositories_path, template_name) recipe_path = Pathname.new(Kameleon.env.workspace).join(recipe_name).to_s begin tpl = Kameleon::RecipeTemplate.new(template_path) tpl.resolve! :strict => false rescue raise if Kameleon.ui.level("verbose") raise TemplateNotFound, "Template '#{template_name}' invalid (try" \ " --verbose) or not found. To see all templates, run the command "\ "`kameleon template list`" else tpl.all_files.each do |path| relative_path = path.relative_path_from(Kameleon.env.repositories_path) dst = File.join(Kameleon.env.workspace, relative_path) copy_file(path, dst) chmod(dst, File.stat(path).mode, {:verbose=>false}) end Dir::mktmpdir do |tmp_dir| recipe_temp = File.join(tmp_dir, File.basename(recipe_path)) ## copying recipe File.open(recipe_temp, 'w+') do |file| message="Try and use extend recipe ERB: " extend_yaml_erb_list = Pathname.new(template_name).dirname.ascend.to_a.map do |p| Kameleon.env.repositories_path.join(p, Kameleon.default_values[:extend_yaml_erb]) end extend_yaml_erb_list.unshift(Kameleon.env.repositories_path.join(template_name.gsub(%r{^(.+?/)?([^/]+?)(\.yaml)?$},'\1.\2') + Kameleon.default_values[:extend_yaml_erb])) extend_yaml_erb_list.push(Pathname.new(Kameleon.erb_dirpath).join("extend.yaml.erb")) extend_yaml_erb = extend_yaml_erb_list.find do |f| Kameleon.ui.verbose(message + f.to_s) message = "-> Not found, fallback: " File.readable?(f) end Kameleon.ui.debug("Open ERB file: '#{extend_yaml_erb}'") result = ERB.new(File.open(extend_yaml_erb, 'rb') { |f| f.read }).result(binding) file.write(result) end copy_file(recipe_temp, recipe_path) end end end desc "info ", "Display detailed information about a recipe" method_option :global, :type => :hash, :default => {}, :aliases => "-g", :desc => "Set custom global variables" method_option :from_cache, :type => :string, :default => nil, :desc => "Get info from a persistent cache tar file (ignore recipe path)" method_option :relative, :type => :boolean, :default => false, :desc => "Make pathnames relative to the current working directory" def info(*recipe_paths) if recipe_paths.empty? if options[:from_cache].nil? raise ArgumentError else unless File.file?(options[:from_cache]) raise CacheError, "The specified cache file "\ "\"#{options[:from_cache]}\" do not exists" end Kameleon.ui.info("Using the cached recipe") @cache = Kameleon::Persistent_cache.instance @cache.cache_path = options[:from_cache] end else recipe_paths.each do |path| recipe = Kameleon::Recipe.new(path) recipe.resolve! recipe.display_info(options[:relative]) end end end desc "dag [ [<...>]]", "Draw a DAG of the steps to build one or more recipes" method_option :global, :type => :hash, :default => {}, :aliases => "-g", :desc => "Set custom global variables" method_option :file, :type => :string, :default => "/tmp/kameleon.dag", :desc => "DAG output filename" method_option :format, :type => :string, :desc => "DAG GraphViz format" method_option :relative, :type => :boolean, :default => false, :desc => "Make pathnames relative to the current working directory" method_option :recipes_only, :type => :boolean, :default => false, :desc => "Show recipes only (mostly useful to display multiple recipes inheritance)" def dag(*recipe_paths) raise ArgumentError if recipe_paths.empty? color = 0 recipes_dag = nil recipe_paths.each do |path| recipe = Kameleon::Recipe.new(path) recipes_dag = Kameleon::Engine.new(recipe, options.dup.merge({no_create_build_dir: true}).freeze).dag(recipes_dag, color, options[:recipes_only]) color += 1 end format = "canon" if options[:format] if GraphViz::Constants::FORMATS.include?(options[:format]) format = options[:format] else Kameleon.ui.warn("Unknown GraphViz format #{options[:format]}, fall back to #{format}") end else options[:file].match(/^.+\.([^\.]+)$/) do |f| if GraphViz::Constants::FORMATS.include?(f[1]) format = f[1] end end end recipes_dag.output( :"#{format}" => options[:file] ) Kameleon.ui.info("Generated GraphViz #{format} file: #{options[:file]}") end desc "export ", "Export the given recipe with its steps and data to a given directory" method_option :global, :type => :hash, :default => {}, :aliases => "-g", :desc => "Set custom global variables" method_option :add, :type => :boolean, :default => false, :aliases => "-A", :desc => "export recipe and steps to an existing directory (this may overwrite some existing files)" def export(recipe_path,dest_path) unless recipe_path.end_with? '.yaml' recipe_path = recipe_path + '.yaml' end # Manage global as it is not passed to env by default if options[:global] Kameleon.env.global.merge!(options[:global]) end recipe = Recipe.new(recipe_path) recipe.resolve! :strict => false recipe.all_files.uniq.each do |path| relative_path = path.relative_path_from(Kameleon.env.workspace) if relative_path.fnmatch("../*") raise if Kameleon.ui.level("verbose") raise ExportError, "Recipe '#{recipe_path}' depends on a file" \ " outside of the current directory: '#{relative_path.to_s}'" end end Kameleon.ui.info("Export recipe #{recipe_path} to directory: #{dest_path}") if File.exist?(dest_path) unless options[:add] raise if Kameleon.ui.level("verbose") raise ExportError, "Target export directory '#{dest_path}' already "\ "exists, use the --add option if you really want to export the "\ "recipe files to it (this may overwrite some existing files)" end else FileUtils.mkdir_p(dest_path) end recipe.all_files.uniq.each do |path| relative_path = path.relative_path_from(Kameleon.env.workspace) dst = File.join(dest_path, relative_path) copy_file(path, dst) end end desc "build ", "Build the appliance from the given recipe" method_option :build_path, :type => :string, :default => nil, :aliases => "-b", :desc => "Set the build directory path" method_option :clean, :type => :boolean, :default => false, :desc => "Run the command `kameleon clean` first" method_option :dryrun, :type => :boolean, :aliases => "-d", :default => false, :desc => "Dry run, only show what would run" method_option :relative, :type => :boolean, :default => false, :desc => "Make dryrun show pathnames relative to the current working directory" method_option :from_checkpoint, :type => :string, :aliases => "-F", :default => nil, :desc => "Restart the build from a specific checkpointed step, instead of the latest one" method_option :begin_checkpoint, :type => :string, :aliases => "-B", :default => nil, :desc => "Only create checkpoints after the given step" method_option :end_checkpoint, :type => :string, :aliases => "-E", :default => nil, :desc => "Do not create checkpoints after the given step" method_option :enable_checkpointing, :type => :boolean, :aliases => "-c", :default => false, :desc => "Enable creating and using checkpoints" method_option :microstep_checkpoints, :type => :string, :enum => ["first", "all"], :default => "all", :desc => "Create checkpoint of the first microstep only, or all" method_option :list_checkpoints, :type => :boolean, :aliases => "-l", :default => false, :desc => "List availables checkpoints" method_option :enable_cache, :type => :boolean, :aliases => "-C", :default => false, :desc => "Generate a persistent cache for the appliance" method_option :cache_path, :type => :string, :default => nil, :desc => "Set the cache directory path" method_option :from_cache, :type => :string, :default => nil, :desc => "Use a persistent cache tar file to build the image" method_option :cache_archive_compression, :type => :string, :enum => ["none", "gzip", "bz2", "xz"], :default => "gzip", :desc => "Set the persistent cache tar file compression" method_option :polipo_path, :type => :string, :default => nil, :desc => "Full path of the polipo binary to use for the persistent cache" method_option :proxy, :type => :string, :default => "", :desc => "HTTP proxy address and port (expected format is hostname:port)" method_option :proxy_credentials, :type => :string, :default => "", :desc => "Username and password if required by the parent proxy (expected format is username:password)" method_option :proxy_offline, :type => :boolean, :default => false, :desc => "Prevent Polipo from contacting remote servers" method_option :global, :type => :hash, :default => {}, :aliases => "-g", :desc => "Set custom global variables" def build(recipe_path=nil) if recipe_path.nil? && !options[:from_cache].nil? unless File.file?(options[:from_cache]) raise CacheError, "The specified cache file "\ "\"#{options[:from_cache]}\" do not exists" end Kameleon.ui.info("Using the cached recipe") @cache = Kameleon::Persistent_cache.instance @cache.cache_path = options[:from_cache] recipe_path = @cache.get_recipe end raise BuildError, "A recipe file or a persistent cache archive " \ "is required to run this command." if recipe_path.nil? if options[:dryrun] Kameleon::Engine.new(Kameleon::Recipe.new(recipe_path), options.dup.merge({no_create_build_dir: true}).freeze).dryrun elsif options[:clean] opts = Hash.new.merge options opts[:lazyload] = false opts[:fail_silently] = true engine = Kameleon::Engine.new(Recipe.new(recipe_path), opts) engine.clean(:with_checkpoint => true) elsif options[:list_checkpoints] Kameleon.ui.level = "error" engine = Kameleon::Engine.new(Recipe.new(recipe_path), options) engine.pretty_checkpoints_list else engine = Kameleon::Engine.new(Recipe.new(recipe_path), options) Kameleon.ui.info("Starting build recipe '#{recipe_path}'") start_time = Time.now.to_i engine.build total_time = Time.now.to_i - start_time Kameleon.ui.info("") Kameleon.ui.info("Successfully built '#{recipe_path}'") Kameleon.ui.info("Total duration: #{total_time} secs") end end desc "commands", "List all available commands", :hide => true def commands(context="main") Kameleon.ui.debug("Commands for '#{context}':") case context when "main" puts Main.all_commands.keys - ["commands"] when "repository" invoke CLI::Repository, "commands", [], [] when "template" invoke CLI::Template, "commands", [], [] end end desc "source_root", "Print the kameleon directory path", :hide => true def source_root puts Kameleon.source_root end map %w(-v --version) => :version map %w(ls) => :list map %w(completions) => :commands def initialize(*args) super self.options ||= {} Kameleon.env = Kameleon::Environment.new(self.options) if !$stdout.tty? or !options["color"] Thor::Base.shell = Thor::Shell::Basic end Kameleon.ui = Kameleon::UI::Shell.new(self.options) if (self.options["debug"] or ENV['KAMELEON_DEBUG']) Kameleon.ui.level = "debug" elsif self.options["verbose"] Kameleon.ui.level = "verbose" end Kameleon.ui.verbose("The level of output is set to #{Kameleon.ui.level}") end def self.start(*args) # `kameleon build -h` does not work without the following, except for subcommands... # Ref: https://stackoverflow.com/a/49044225/6431461 if (Thor::HELP_MAPPINGS & ARGV).any? and subcommands.grep(/^#{ARGV[0]}/).empty? Kameleon.ui.debug("Apply workaround to handle the help command in #{ARGV}") Thor::HELP_MAPPINGS.each do |cmd| if match = ARGV.delete(cmd) ARGV.unshift match end end end super rescue Exception => e Kameleon.ui = Kameleon::UI::Shell.new raise e end end end kameleon-builder-2.11.0/lib/kameleon/utils.rb0000644000004100000410000002063015005702337021073 0ustar www-datawww-data# The progressbar and ruby-progressbar gems are the same code from # https://github.com/jfelchner/ruby-progressbar/tree/master # Only the lib head file is named progressbar.rb vs. ruby-progressbar.rb. # The progressbar gem is the one packaged as ruby-progressbar in Debian. # Using that one (see gemspec) and require "progressbar" not "ruby-progessbar". require 'progressbar' module Kameleon module Utils @@warned_vars = Array.new def self.warn_var(var) if ! @@warned_vars.include?(var) Kameleon.ui.warn("Warning: variable $$#{var[0]} is not enclosed with braces, which may cause errors. Please prefer using $${#{var[0]}}.") @@warned_vars.push(var) end end def self.resolve_vars(raw, yaml_path, initial_variables, recipe, kwargs = {}) raw = resolve_data_dir_vars(raw, yaml_path, initial_variables, recipe, kwargs) return resolve_simple_vars(raw, yaml_path, initial_variables, kwargs) end def self.resolve_data_dir_vars(raw, yaml_path, initial_variables, recipe, kwargs) raw.to_s.scan(/\$\$kameleon\_data\_dir\/(.*)/) do |var| warn_var(var) end reg = %r/(\$\$kameleon\_data\_dir|\$\${kameleon\_data\_dir})(.*)/ matches = raw.to_enum(:scan, reg).map { Regexp.last_match } matches.each do |m| unless m.nil? path = resolve_simple_vars(m[2], yaml_path, initial_variables, kwargs) resolved_path = recipe.resolve_data_path(path.chomp('"'), yaml_path) raw.gsub!(m[0].chomp('"'), "#{resolved_path}") end end return raw end def self.resolve_simple_vars_once(raw, initial_variables) raw.to_s.scan(/\$\$([a-zA-Z0-9\-_]+)/) do |var| warn_var(var) end raw.to_s.gsub(/\$\$\{[a-zA-Z0-9\-_]+\}|\$\$[a-zA-Z0-9\-_]+/) do |var| # remove the dollars if var.include? "{" strip_var = var[3,(var.length - 4)] else strip_var = var[2,(var.length - 2)] end # check in local vars if initial_variables.has_key? strip_var value = initial_variables[strip_var] end return $` + value.to_s + $' end end # Variables are replaced correctly for recursive variable overload of # the parent by the child: # For example: # Parent={var: 10} # Child={var: $$var 11} # => {var: 10 11} def self.overload_merge(parent_dict, child_dict) parent_dict.merge(child_dict){ |key, old_value, new_value| if new_value.to_s.include?("$$" + key.to_s) or new_value.to_s.include?("$${" + key.to_s + "}") Utils.resolve_simple_vars_once(new_value, {key => old_value}) else new_value end } end def self.resolve_simple_vars(raw, yaml_path, initial_variables, kwargs) raw.to_s.scan(/\$\$([a-zA-Z0-9\-_]+)/) do |var| warn_var(var) end raw.to_s.gsub(/\$\$\{[a-zA-Z0-9\-_]+\}|\$\$[a-zA-Z0-9\-_]+/) do |var| # remove the dollars if var.include? "{" strip_var = var[3,(var.length - 4)] else strip_var = var[2,(var.length - 2)] end # check in local vars if initial_variables.has_key? strip_var value = initial_variables[strip_var] Kameleon.ui.debug("Resolved variable = #{strip_var}: #{value}") else if kwargs.fetch(:strict, true) fail RecipeError, "#{yaml_path}: variable #{var} not found in local or global" end end return $` + resolve_simple_vars(value.to_s + $', yaml_path, initial_variables, kwargs) end end def self.generate_slug(str) value = str.strip value.gsub!(/['`]/, "") value.gsub!(/\s*@\s*/, " at ") value.gsub!(/\s*&\s*/, " and ") value.gsub!(/\s*[^A-Za-z0-9\.]\s*/, '_') value.gsub!(/_+/, "_") value.gsub!(/\A[_\.]+|[_\.]+\z/, "") value.chomp("_").downcase end def self.extract_meta_var(name, content) start_regex = Regexp.escape("# #{name.upcase}: ") end_regex = Regexp.escape("\n#\n") reg = %r/#{ start_regex }(.*?)#{ end_regex }/m var = content.match(reg).captures.first var.gsub!("\n#", "") var.gsub!(" ", " ") return var rescue end def self.copy_files(relative_dir, dest_dir, files2copy) files2copy.each do |path| relative_path = path.relative_path_from(relative_dir) dst = File.join(dest_dir,relative_path) FileUtils.mkdir_p File.dirname(dst) FileUtils.copy_file(path, dst) end end def self.list_recipes(recipes_path, filter, do_progressbar = false, is_repository = false, kwargs = {}) Kameleon.env.root_dir = recipes_path catch_exception = kwargs.fetch(:catch_exception, true) recipes_hash = [] recipes_files = get_recipes(recipes_path, filter) if recipes_files.empty? Kameleon.ui.shell.say " ", :cyan return end if do_progressbar progressbar = ProgressBar.create(:format => '%t (%p%%) %bᗧ%i', :title => 'Resolving ' + if is_repository; 'templates' else 'recipes' end, :progress_mark => '.', :remainder_mark => '･', :total => recipes_files.size + 10, :starting_at => 10) end recipes_files.each do |f| path = f.to_s begin recipe = RecipeTemplate.new(path) name = path.gsub(recipes_path.to_s + '/', '').chomp('.yaml') recipes_hash.push({ "name" => name, "description" => recipe.metainfo['description'], }) progressbar.increment if do_progressbar rescue => e raise e if Kameleon.env.debug or not catch_exception end end unless recipes_hash.empty? name_width = recipes_hash.map { |k| k['name'].size }.max desc_width = Thor::Shell::Terminal.terminal_width - name_width - 3 desc_width = (80 - name_width - 3) if desc_width < 0 end repo_str_old = nil recipes_hash.each do |r| if is_repository repo_str,recipe_dir_str,recipe_str = r["name"].match(%r{^([^/]+/)(.+/)?([^/]+)$}).to_a[1..3].map{|m| m.to_s} else repo_str,recipe_dir_str,recipe_str = r["name"].match(%r{^()(.+/)?([^/]+)$}).to_a[1..3].map{|m| m.to_s} end if not repo_str_old.nil? and repo_str_old != repo_str Kameleon.ui.shell.say "#{'-' * name_width} | #{'-' * desc_width}" end repo_str_old = repo_str Kameleon.ui.debug("#{r["name"]} -> repo=#{repo_str}, recipe_dir=#{recipe_dir_str}, recipee=#{recipe_str}") Kameleon.ui.shell.say "#{repo_str}", :yellow, false Kameleon.ui.shell.say "#{recipe_dir_str}", :cyan, false Kameleon.ui.shell.say sprintf("%-#{name_width - repo_str.length - recipe_dir_str.length}s", recipe_str), :magenta, false Kameleon.ui.shell.say " | ", nil, false if r["description"].to_s.length > desc_width - 4 r["description"] = r["description"][0..(desc_width - 4)] + "..." end Kameleon.ui.shell.say r["description"], :blue end end def self.get_recipes(path, filter = nil, base = nil) base = path if base.nil? if filter.nil? begin filter = File.read(File.join(path, "/.filter")).chomp Kameleon.ui.verbose("Found filter #{filter} in #{path}") base = path rescue end end recipes = path.children.select{|child| child.file? and child.extname == ".yaml"}.map do |child| recipe = child.to_s.gsub(base.to_s + '/', '').chomp('.yaml') if filter.nil? or Regexp.new(filter).match(recipe) child else Kameleon.ui.verbose("Filters out #{recipe}, does not match #{filter}") nil end end.select { |x| x }.flatten(1).sort{|a,b| a.to_s <=> b.to_s} recipes + path.children.select{|child| child.directory? and child.basename.to_s != "steps" and child.basename.to_s != ".steps"}.sort{|a,b| a.to_s <=> b.to_s}.map do |child| get_recipes(child, filter, base) end.flatten(1) end def self.which(cmd) ENV['PATH'].split(File::PATH_SEPARATOR).each do |path| exe = File.join(path, "#{cmd}") return path if File.executable? exe end return nil end end end kameleon-builder-2.11.0/lib/kameleon/context.rb0000644000004100000410000001127115005702337021420 0ustar www-datawww-datarequire 'kameleon/shell' module Kameleon class Context attr_accessor :shell attr_accessor :name attr_accessor :cmd attr_accessor :interactive_cmd attr_accessor :workdir attr_accessor :local_workdir attr_accessor :proxy attr_accessor :env_files def initialize(name, cmd, interactive_cmd, workdir, exec_prefix, local_workdir, env_files, kwargs = {}) @name = name.downcase @cmd = cmd @interactive_cmd = interactive_cmd @workdir = workdir @exec_prefix = exec_prefix @local_workdir = local_workdir @proxy = kwargs[:proxy] @fail_silently = kwargs.fetch(:fail_silently, true) @lazyload = kwargs.fetch(:lazyload, false) @env_files = env_files @shell = Kameleon::Shell.new(self) @already_loaded = false Kameleon.ui.debug("Initialize new ctx (#{name})") instance_variables.each do |v| Kameleon.ui.debug(" #{v} = #{instance_variable_get(v)}") end @cache = Kameleon::Persistent_cache.instance unless @lazyload load_shell end end def already_loaded? @already_loaded end def do_log(out, log_level) prefix = "[#{@name}] " out.gsub!("\r", "\r#{prefix}") out.gsub!("\n", "\n#{prefix}") if Kameleon.log_on_progress if out.end_with?("#{prefix}") Kameleon.log_on_progress = false log_progress(log_level, out.chomp(prefix)) else log_progress(log_level, out) end else if out.end_with?("#{prefix}") log_progress(log_level, prefix + out.chomp(prefix)) else Kameleon.log_on_progress = true log_progress(log_level, prefix + out) end end end def log_progress(log_level, msg) Kameleon.ui.confirm(msg, false) if log_level == "info" Kameleon.ui.error(msg, false) if log_level == "error" Kameleon.ui.verbose msg if log_level == "verbose" Kameleon.ui.debug msg if log_level == "debug" end def execute(cmd, kwargs = {}) load_shell cmd_with_prefix = "#{@exec_prefix} #{cmd}" log_level = kwargs.fetch(:log_level, "info") exit_status = @shell.execute(cmd_with_prefix, kwargs) do |out, err| do_log(out, log_level) unless out.nil? do_log(err, "error") unless err.nil? end Kameleon.ui.verbose("Exit status: #{exit_status}") fail ExecError unless exit_status.eql? 0 rescue ShellError, Errno::EPIPE => e Kameleon.ui.verbose("Shell cmd failed to launch: #{@shell.shell_cmd}") raise ExecError, e.message + ". The '#{@name}_context' is inaccessible." rescue ShellExited raise ContextClosed, "The ctx '#{name}' was closed" end def pipe(cmd, other_cmd, other_ctx, kwargs = {}) if @cache.mode == :from then Kameleon.ui.info("Redirecting pipe into cache") tmp = @cache.get_cache_cmd(cmd) else tmp = Tempfile.new("pipe-#{ Kameleon::Utils.generate_slug(cmd)[0..20] }") Kameleon.ui.verbose("Running piped commands") Kameleon.ui.verbose("Saving STDOUT from #{@name}_ctx to local file #{tmp.path}") execute(cmd, kwargs.merge({:stdout => tmp})) tmp.close end ## Saving one side of the pipe into the cache if @cache.mode == :build then @cache.cache_cmd(cmd,tmp.path) end Kameleon.ui.verbose("Forwarding #{tmp.path} to STDIN of #{other_ctx.name}_ctx") dest_pipe_path = "${KAMELEON_WORKDIR}/pipe-#{ Kameleon::Utils.generate_slug(other_cmd)[0..20] }" other_ctx.load_shell other_ctx.send_file(tmp.path, dest_pipe_path) other_cmd_with_pipe = "cat #{dest_pipe_path} | #{other_cmd} && rm #{dest_pipe_path}" other_ctx.execute(other_cmd_with_pipe, kwargs) end def load_shell() unless @shell.started? || @shell.exited? @shell.restart execute("echo The '#{name}_context' has been initialized", :log_level => "info") @already_loaded = true end rescue Exception => e @shell.stop if @fail_silently e.message.split( /\r?\n/ ).each {|m| Kameleon.ui.error m } else raise end end def start_shell #TODO: Load env and history load_shell Kameleon.ui.info("Starting interactive shell") @shell.start_interactive rescue ShellError => e e.message.split( /\r?\n/ ).each {|m| Kameleon.ui.error m } end def closed? return !@shell.started? || @shell.exited? end def close! @shell.stop end def reload @shell = Kameleon::Shell.new(self) @shell.start end def send_file(source_path, dest_path) @shell.send_file(source_path, dest_path) end end end kameleon-builder-2.11.0/lib/kameleon/ui.rb0000644000004100000410000000631615005702337020355 0ustar www-datawww-datamodule Kameleon module UI class Silent def info(message, newline = nil) end def confirm(message, newline = nil) end def warn(message, newline = nil) end def error(message, newline = nil) end def debug(message, newline = nil) end def debug? false end def quiet? false end def ask(message) end def level=(name) end def level(name = nil) end def trace(message, newline = nil) end def silence yield end end class Shell LEVELS = %w(silent error warn confirm info verbose debug) attr_accessor :shell def initialize(options = {}) @shell = Thor::Base.shell.new @level = ENV['KAMELEON_DEBUG'] ? "debug" : "info" end def info(msg, newline = nil) tell_me(msg, nil, newline) if level("info") end def msg(msg, newline = nil) tell_me(msg, :blue, newline) if level("info") end def confirm(msg, newline = nil) tell_me(msg, :green, newline) if level("confirm") end def warn(msg, newline = nil) tell_me(msg, :yellow, newline) if level("warn") end def error(msg, newline = nil) tell_me(msg, :red, newline) if level("error") end def verbose(msg, newline = nil) tell_me("[info] #{msg}", nil, newline) if level("verbose") end def debug(msg, newline = nil) tell_me("[debug] #{msg}", nil, newline) if level("debug") end def debug? # needs to be false instead of nil to be newline param to other methods level("debug") end def quiet? LEVELS.index(@level) <= LEVELS.index("warn") end def ask(msg) @shell.ask(msg) end def level=(level) raise ArgumentError unless LEVELS.include?(level.to_s) @level = level end def level(name = nil) name ? LEVELS.index(name) <= LEVELS.index(@level) : @level end def trace(e, newline = nil) msg = ["#{e.class}: #{e.message}", *e.backtrace].join("\n") if debug? tell_me(msg, nil, newline) elsif @trace STDERR.puts "#{msg}#{newline}" end end def silence old_level, @level = @level, "silent" yield ensure @level = old_level end private # valimism def tell_me(msg, color = nil, newline = nil) msg = word_wrap(msg) if newline.is_a?(Hash) && newline[:wrap] if newline.nil? if Kameleon.log_on_progress Kameleon.log_on_progress = false msg = "\n" + msg end @shell.say(msg, color) else @shell.say(msg, color, newline) end end def strip_leading_spaces(text) spaces = text[/\A\s+/, 0] spaces ? text.gsub(/#{spaces}/, '') : text end def word_wrap(text, line_width = Thor::Shell::Terminal.terminal_width) strip_leading_spaces(text).split("\n").collect do |line| line.length > line_width ? line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip : line end * "\n" end end end end kameleon-builder-2.11.0/lib/kameleon/environment.rb0000644000004100000410000000336715005702337022307 0ustar www-datawww-datamodule Kameleon # This class allows access to the recipes, CLI, etc. all in the scope of # this environment class Environment def script? @script end def initialize(options = {}) # symbolify commandline options options = options.inject({}) {|result,(key, value)| result.update({key.to_sym => value})} workspace = File.expand_path(Dir.pwd) # templates_path = File.expand_path(options[:templates_path] || Kameleon.default_templates_path) build_path = File.expand_path(options[:build_path] || File.join(workspace, "build")) cache_path = File.expand_path(options[:cache_path] || File.join(build_path, "cache")) repositories_path = File.expand_path(Kameleon.default_values[:repositories_path]) env_options = { :workspace => Pathname.new(workspace), # :templates_path => Pathname.new(templates_path), :build_path => Pathname.new(build_path), :cache_path => Pathname.new(cache_path), :repositories_path => Pathname.new(repositories_path), :root_dir => Pathname.new(workspace), :global => options.fetch(:global, {}), } options = Kameleon.default_values.merge(options).merge(env_options) Kameleon.ui.debug("Environment initialized (#{self})") # Injecting all variables of the options and assign the variables options.each do |key, value| self.class.__send__(:attr_accessor, "#{key}") instance_variable_set("@#{key}".to_sym, options[key]) Kameleon.ui.debug(" @#{key} : #{options[key]}") end @debug = true if ENV['KAMELEON_DEBUG'] unless (File.exist?(@repositories_path.to_s) || File.symlink?(@repositories_path.to_s)) Dir.mkdir(@repositories_path.to_s) end end end end kameleon-builder-2.11.0/lib/kameleon/version.rb0000644000004100000410000000005115005702337021413 0ustar www-datawww-datamodule Kameleon VERSION = '2.11.0' end kameleon-builder-2.11.0/lib/kameleon/recipe.rb0000644000004100000410000010021215005702337021175 0ustar www-datawww-datarequire 'kameleon/utils' require 'kameleon/step' module Kameleon class Recipe attr_accessor :path attr_accessor :name attr_accessor :global attr_accessor :sections attr_accessor :aliases attr_accessor :aliases_path attr_accessor :checkpoint attr_accessor :checkpoint_path attr_accessor :metainfo attr_accessor :files attr_accessor :extended_recipe_file attr_accessor :base_recipes_files attr_accessor :data_files attr_accessor :env_files attr_accessor :cli_global def initialize(path, kwargs = {}) @path = Pathname.new(File.expand_path(path)) if not @path.exist? and @path.extname != ".yaml" @path = Pathname.new(File.expand_path(path + ".yaml")) end @name = (@path.basename ".yaml").to_s @recipe_content = File.open(@path, 'r') { |f| f.read } @sections = { "bootstrap" => Section.new("bootstrap"), "setup" => Section.new("setup"), "export" => Section.new("export"), } @cli_global = Kameleon.env.global.clone @cli_global.each do |k,v| Kameleon.ui.warn("CLI Global variable override: #{k} => #{v}") end @global = { "kameleon_recipe_name" => @name, "kameleon_recipe_dir" => File.dirname(@path), "kameleon_cwd" => File.join(Kameleon.env.build_path, @name), "in_context" => {"cmd" => "/bin/bash", "proxy_cache" => "127.0.0.1"}, "out_context" => {"cmd" => "/bin/bash", "proxy_cache" => "127.0.0.1"}, "proxy_local" => "", "proxy_out" => "", "proxy_in" => "", "checkpointing_enabled" => "false", "persistent_cache" => "false", } @aliases = {} @checkpoint = nil @step_files = [] Kameleon.ui.verbose("Initialize new recipe (#{path})") @base_recipes_files = [@path] @data_files = [] @env_files = [] @steps_dirs = [] load! :strict => false end def update_steps_dirs() # Where we can find steps @steps_dirs = @base_recipes_files.map do |recipe_path| get_steps_dirs(recipe_path) end.flatten! end def get_steps_dirs(recipe_path) relative_path = recipe_path.to_s.gsub(Kameleon.env.root_dir.to_s + '/', '') if relative_path.eql? recipe_path.to_s subdirs = [recipe_path.dirname] else last_dir = Kameleon.env.root_dir subdirs = [last_dir] relative_path.split("/")[0...-1].each do |p| subdir = last_dir.join(p) subdirs.push(subdir) last_dir = subdir end end steps_dirs = [] subdirs.reverse_each do |p| steps_dirs.push(File.expand_path(File.join(p.to_s, 'steps'))) steps_dirs.push(File.expand_path(File.join(p.to_s, '.steps'))) end steps_dirs.select! { |x| File.exist? x } end def load!(kwargs = {}) # Find recipe path Kameleon.ui.verbose("Loading #{@path}") fail RecipeError, "Could not find this following recipe: #{@path}" \ unless File.file? @path yaml_recipe = YAML.unsafe_load_file @path unless yaml_recipe.kind_of? Hash fail RecipeError, "Invalid yaml: #{@path}" end update_steps_dirs() extended_recipe_name = yaml_recipe.fetch("extend", "") unless extended_recipe_name.nil? extended_recipe_name << ".yaml" unless extended_recipe_name.end_with? ".yaml" @extended_recipe_file = Pathname.new(File.expand_path(File.join(File.dirname(path), extended_recipe_name))) end # Load extended recipe variables yaml_recipe = load_base_recipe(yaml_recipe, @path) yaml_recipe.delete("extend") # Where we can find steps @steps_dirs = @base_recipes_files.map do |recipe_path| get_steps_dirs(recipe_path) end.flatten! @steps_dirs.uniq! # Set default value for in_ctx and out_ctx options %w(out_context in_context).each do |context_name| unless yaml_recipe.keys.include? "global" yaml_recipe["global"] = {} end unless yaml_recipe["global"].keys.include? context_name yaml_recipe["global"][context_name] = {} end @global[context_name].merge!(yaml_recipe["global"][context_name]) yaml_recipe["global"][context_name] = @global[context_name] unless yaml_recipe["global"][context_name].keys.include? "interactive_cmd" yaml_recipe["global"][context_name]["interactive_cmd"] = yaml_recipe["global"][context_name]['cmd'] end end # Load Global variables @global.merge!(yaml_recipe.fetch("global", {})) # merge cli variable with recursive variable overload @global = Utils.overload_merge(@global, @cli_global) # Resolve dynamically-defined variables !! resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global, self, kwargs) resolved_global = @global.merge YAML.unsafe_load(resolved_global) Kameleon.ui.debug("Resolved_global: #{resolved_global}") # Loads aliases load_aliases(yaml_recipe) # Load env files load_env_files(yaml_recipe) # Loads checkpoint configuration load_checkpoint_config(yaml_recipe) include_steps = resolved_global['include_steps'] include_steps ||= [] include_steps.push '' include_steps.flatten! include_steps.compact! Kameleon.ui.debug("include steps: #{include_steps}") @sections.values.each do |section| dir_to_search = @steps_dirs.map do |steps_dir| include_steps.map do |path| [File.join(steps_dir, section.name, path), File.join(steps_dir, path)] end end.flatten.select { |x| File.exist? x } Kameleon.ui.debug("Directory to search for steps: #{dir_to_search}") if yaml_recipe.key? section.name yaml_section = yaml_recipe.fetch(section.name) next unless yaml_section.kind_of? Array yaml_section.each do |raw_macrostep| embedded_step = false # Get macrostep name and arguments if available if raw_macrostep.kind_of? String name = Utils.resolve_vars(raw_macrostep, @path, @global, self, kwargs) args = nil elsif raw_macrostep.kind_of? Hash name = Utils.resolve_vars(raw_macrostep.keys[0], @path, @global, self, kwargs) args = raw_macrostep.values[0] else fail RecipeError, "Malformed yaml recipe in section: "\ "#{section.name}" end # Detect if step is embedded if not args.nil? args.each do |arg| if arg.kind_of? Hash if arg[arg.keys[0]].kind_of? Array embedded_step = true end end end end if embedded_step Kameleon.ui.verbose("Loading embedded macrostep #{name}") macrostep = load_macrostep(nil, name, args, kwargs) section.macrosteps.push(macrostep) next end # Load macrostep yaml loaded = false dir_to_search.each do |dir| macrostep_path = Pathname.new(File.join(dir, name + '.yaml')) if File.file?(macrostep_path) Kameleon.ui.verbose("Loading macrostep #{macrostep_path}") macrostep = load_macrostep(macrostep_path, name, args, kwargs) section.macrosteps.push(macrostep) @step_files.push(macrostep_path) Kameleon.ui.verbose("Macrostep '#{name}' found in this path: " \ "#{macrostep_path}") loaded = true break else Kameleon.ui.verbose("Macrostep '#{name}' not found in this path: " \ "#{macrostep_path}") end end fail RecipeError, "Step #{name} not found" unless loaded end end end Kameleon.ui.verbose("Loading recipe metadata") @metainfo = { "description" => Utils.extract_meta_var("description", @recipe_content) } end def load_base_recipe(yaml_recipe, path) base_recipe_name = yaml_recipe.fetch("extend", "") return yaml_recipe if base_recipe_name.empty? # resolve variable in extends to permit backend selection base_recipe_name = Utils.resolve_simple_vars_once( base_recipe_name, Utils.overload_merge(load_global(yaml_recipe, path), @cli_global)) ## check that the recipe has not already been loaded base_recipe_name << ".yaml" unless base_recipe_name.end_with? ".yaml" base_recipe_path = File.join(File.dirname(path), base_recipe_name) ## check that the recipe has not already been loaded return yaml_recipe if @base_recipes_files.include? base_recipe_path @base_recipes_files.push(Pathname.new(File.expand_path(base_recipe_path))) update_steps_dirs() base_recipe_path << ".yaml" unless base_recipe_path.end_with? ".yaml" fail RecipeError, "Could not find this following recipe: #{@recipe_path}" \ unless File.file? path base_yaml_recipe = YAML.unsafe_load_file base_recipe_path unless yaml_recipe.kind_of? Hash fail RecipeError, "Invalid yaml: #{base_yaml_recipe}" end base_yaml_recipe.keys.each do |key| if ["export", "bootstrap", "setup"].include? key base_yaml_recipe.delete(key) unless yaml_recipe.keys.include? key end end yaml_recipe.keys.each do |key| if ["aliases", "checkpoint", "env"].include? key base_yaml_recipe[key] = yaml_recipe[key] elsif ["export", "bootstrap", "setup"].include? key base_section = base_yaml_recipe.fetch(key, []) base_section = [] if base_section.nil? recipe_section = yaml_recipe[key] recipe_section = [] if recipe_section.nil? index_base_steps = recipe_section.index("@base") unless index_base_steps.nil? recipe_section[index_base_steps] = base_section recipe_section.flatten! end base_yaml_recipe[key] = recipe_section elsif ["global"].include? key base_section = load_global(base_yaml_recipe, base_recipe_path) recipe_section = load_global(yaml_recipe, path) # manage recursive variable overload base_yaml_recipe[key] = Utils.overload_merge(base_section, recipe_section) end end return load_base_recipe(base_yaml_recipe, base_recipe_path) end def load_global(yaml_recipe, recipe_path) global = {} if yaml_recipe.keys.include? "global" global_loaded = yaml_recipe.fetch("global", {}) global_loaded = {} if global_loaded.nil? if global_loaded.kind_of? Hash global_loaded.each do |key, value| if key.eql? "include" global_to_include = load_include_global(value, recipe_path) global.merge!(global_to_include) else global[key] = value end end end end return global end def load_include_global(yaml_include, recipe_path) def load_global_file(global_file, recipe_path) def try_to_load(absolute_path) if File.file?(absolute_path) global_to_include = YAML.unsafe_load_file(absolute_path) if global_to_include.kind_of? Hash @step_files.push(absolute_path) return global_to_include else fail RecipeError, "Global should be a Hash. (check #{absolute_path})" end end end ## check that the recipe has not already been loaded global_file << ".yaml" unless global_file.end_with? ".yaml" dir_search = @steps_dirs.map do |steps_dir| File.join(steps_dir, "global") end.flatten dir_search.unshift(File.join(File.dirname(recipe_path))) # try relative/absolute path if Pathname.new(global_file).absolute? global_to_include = try_to_load(global_file) unless global_to_include.nil? return global_to_include else fail RecipeError, "File '#{global_file}' not found" end else dir_search.each do |dir_path| absolute_path = Pathname.new(File.join(dir_path, global_file)) global_to_include = try_to_load(absolute_path) unless global_to_include.nil? return global_to_include end end end rel_dir_search = dir_search.map do |steps_dir| Pathname.new(steps_dir).relative_path_from(Pathname(Dir.pwd)).to_s end.flatten fail RecipeError, "File '#{global_file}' not found here #{rel_dir_search}" end global_hash = {} if yaml_include.kind_of? String list_files = [yaml_include] elsif yaml_include.kind_of? Array list_files = [] yaml_include.each do |value| if value.kind_of? String list_files.push(value) end end else return global_hash end list_files.each do |includes_file| filename = includes_file if includes_file.start_with?("-") filename = includes_file[1..-1] end begin new_global = load_global_file(filename, recipe_path) global_hash.merge!(new_global) rescue unless includes_file.start_with?("-") raise end end end return global_hash end def load_aliases(yaml_recipe) def load_aliases_file(aliases_file) dir_search = @steps_dirs.map do |steps_dir| File.join(steps_dir, "aliases") end.flatten dir_search.each do |dir_path| path = Pathname.new(File.join(dir_path, aliases_file)) if File.file?(path) Kameleon.ui.verbose("Loading aliases #{path}") @aliases.merge!(YAML.unsafe_load_file(path)) @step_files.push(path) return path end end fail RecipeError, "Aliases file for recipe '#{@path}' does not exists" end if yaml_recipe.keys.include? "aliases" aliases = yaml_recipe.fetch("aliases") if aliases.kind_of? Hash @aliases = aliases elsif aliases.kind_of? String load_aliases_file(aliases) elsif aliases.kind_of? Array aliases.each do |aliases_file| load_aliases_file(aliases_file) end end end end def load_env_files(yaml_recipe) def add_env_file(env_file) dir_search = @steps_dirs.map do |steps_dir| File.join(steps_dir, "env") end.flatten dir_search.each do |dir_path| path = Pathname.new(File.join(dir_path, env_file)) if File.file?(path) Kameleon.ui.verbose("Adding env file #{path}") @env_files.push(path) return path end end fail RecipeError, "The env file script '#{env_file}' does not exists "\ "in any of these directories: #{dir_search}" end if yaml_recipe.keys.include? "env" env_content = yaml_recipe.fetch("env") if env_content.kind_of? String add_env_file(env_content) elsif env_content.kind_of? Array env_content.each do |env_file| add_env_file(env_file) end end end end def load_checkpoint_config(yaml_recipe) if yaml_recipe.keys.include? "checkpoint" checkpoint = yaml_recipe.fetch("checkpoint") if checkpoint.kind_of? Hash @checkpoint = checkpoint @checkpoint["path"] = @path elsif checkpoint.kind_of? String dir_search = @steps_dirs.map do |steps_dir| File.join(steps_dir, "checkpoints") end.flatten dir_search.each do |dir_path| path = Pathname.new(File.join(dir_path, checkpoint)) if File.file?(path) Kameleon.ui.verbose("Loading checkpoint configuration #{path}") @checkpoint = YAML.unsafe_load_file(path) @checkpoint["path"] = path.to_s @step_files.push(path) break end end fail RecipeError, "Checkpoint configuraiton file '#{checkpoint}' " \ "does not exists" if @checkpoint.nil? end (@checkpoint.keys - ["path"]).each do |key| @checkpoint[key].map! do |cmd| Kameleon::Command.new(cmd, "checkpoint") end end end end def load_macrostep(step_path, name, args, kwargs) if step_path.nil? macrostep_yaml = args step_path = @path else macrostep_yaml = YAML.unsafe_load_file(step_path) # Basic macrostep syntax check if not macrostep_yaml.kind_of? Array fail RecipeError, "The macrostep #{step_path} is not valid " "(should be a list of microsteps)" end end local_variables = {} loaded_microsteps = [] # Load default local variables macrostep_yaml.each do |yaml_microstep| key = yaml_microstep.keys[0] value = yaml_microstep[key] # Set new variable if not defined yet if value.kind_of? Array loaded_microsteps.push Microstep.new(yaml_microstep) else local_variables[key] = @global.fetch(key, value) end end unless step_path.nil? selected_microsteps = [] if args args.each do |entry| if entry.kind_of? Hash # resolve variable before using it entry.each do |key, value| local_variables[key] = value end elsif entry.kind_of? String selected_microsteps.push entry end end end unless selected_microsteps.empty? # Some steps are selected so remove the others # WARN: Allow the user to define this list not in the original order strip_microsteps = [] selected_microsteps.each do |microstep_name| macrostep = find_microstep(microstep_name, loaded_microsteps) if macrostep.nil? fail RecipeError, "Can't find microstep '#{microstep_name}' "\ "in macrostep file '#{step_path}'" else strip_microsteps.push(macrostep) end end loaded_microsteps = strip_microsteps end end return Macrostep.new(name, loaded_microsteps, local_variables, step_path) end def find_microstep(microstep_name, loaded_microsteps) Kameleon.ui.verbose("Looking for microstep #{microstep_name}") loaded_microsteps.each do |microstep| if microstep_name.eql? microstep.name return microstep end end return nil end def resolve_data_path(partial_path, step_path) Kameleon.ui.verbose("Looking for data '#{partial_path}'") dir_search = @steps_dirs.map do |steps_dir| File.join(steps_dir, "data") end.flatten dir_search.each do |dir_path| real_path = Pathname.new(File.join(dir_path, partial_path)).cleanpath if real_path.exist? Kameleon.ui.verbose("Register data #{real_path}") @data_files.push(real_path) unless @data_files.include? real_path return real_path end Kameleon.ui.verbose("#{real_path} : nonexistent") end fail RecipeError, "Cannot find data '#{partial_path}' used in '#{step_path}'" end def resolve!(kwargs = {}) Kameleon.ui.verbose("Resolving recipe...") unless @global.keys.include? "kameleon_uuid" kameleon_id = SecureRandom.uuid @global["kameleon_uuid"] = kameleon_id @global["kameleon_short_uuid"] = kameleon_id.split("-").last end # Resolve dynamically-defined variables !! resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global, self, kwargs) @global.merge! YAML.unsafe_load(resolved_global) consistency_check resolve_checkpoint unless @checkpoint.nil? Kameleon.ui.verbose("Resolving aliases") @sections.values.each do |section| section.macrosteps.each do |macrostep| # First pass: resolve aliases Kameleon.ui.debug("Resolving aliases for macrostep '#{macrostep.name}'") macrostep.microsteps.each do |microstep| microstep.commands.map! do |cmd| resolve_alias(cmd) end # flatten for multiple-command alias + variables microstep.commands.flatten! end end end Kameleon.ui.verbose("Resolving variables") @sections.values.each do |section| section.macrosteps.each do |macrostep| macrostep.resolve_variables!(@global, self) end end @sections.values.each do |section| section.macrosteps.each do |macrostep| # Second pass: resolve variables + clean/init hooks macrostep.microsteps.each do |microstep| microstep.commands.map! do |cmd| resolve_hooks(cmd, macrostep, microstep) end end Kameleon.ui.debug("Compacting macrostep '#{macrostep.name}'") # remove empty steps macrostep.microsteps.map! do |microstep| microstep.commands.compact! microstep.commands.empty? ? nil : microstep end # remove nil values macrostep.microsteps.compact! Kameleon.ui.debug("Resolving commands for macrostep '#{macrostep.name}'") macrostep.microsteps.each do |microstep| microstep.resolve! end end end calculate_step_identifiers flatten_data Kameleon.ui.verbose("Recipe is resolved") end def consistency_check() # flatten list of hash to an a hash %w(out_context in_context).each do |context_name| if @global[context_name].kind_of? Array old_context_args = @global[context_name].clone @global[context_name] = {} old_context_args.each do |arg| @global[context_name].merge!(arg) end end end Kameleon.ui.verbose("Starting recipe consistency check") # check context args required_args = %w(cmd) missings = [] %w(out_context in_context).each do |context_name| context = @global[context_name] missings = required_args - (context.keys() & required_args) fail RecipeError, "Required paramater missing for #{context_name}:" \ " #{ missings.join ' ' }" unless missings.empty? end unless @checkpoint.nil? required_args = %w(create apply list clear) missings = [] missings = required_args - (@checkpoint.keys() & required_args) fail RecipeError, "Required paramater missing for checkpoint:" \ " #{ missings.join ' ' }" unless missings.empty? end end def resolve_checkpoint() (@checkpoint.keys - ["path"]).each do |key| @checkpoint[key].map! do |cmd| resolve_alias(cmd) end.flatten! @checkpoint[key].each do |cmd| cmd.string_cmd = Utils.resolve_vars(cmd.string_cmd, @checkpoint["path"], @global, self) end end end def resolve_alias(cmd) name = cmd.key if @aliases.keys.include?(name) Kameleon.ui.debug("Resolving alias '#{name}'") aliases_cmd = @aliases.fetch(name).clone aliases_cmd_str = aliases_cmd.to_yaml yaml = YAML.unsafe_load(cmd.string_cmd) args = [] if yaml.is_a?(Hash) # if cmd is an alias with no args, yaml is a String args.push(yaml[name]).flatten! # convert args to array end expected_args_number = aliases_cmd_str.scan(/@\d+/).uniq.count if expected_args_number != args.count msg = if expected_args_number.zero? "#{name} takes no arguments (#{args.count} given)" else "#{name} takes exactly #{expected_args_number} arguments (#{args.count} given)" end raise RecipeError, msg end aliases_cmd.map do |c| nc = Command.new(c, cmd.microstep_name) args.each_with_index do |arg, i| nc.gsub!("@#{i + 1}", arg) end resolve_alias(nc) end elsif cmd.value.is_a?(Array) Kameleon.ui.debug("Search for aliases in the sub-commands of '#{name}'") cmd.value.map! { |c| resolve_alias(c) }.flatten! cmd.remaster_string_cmd_from_value! else Kameleon.ui.debug("Leaf command '#{name}' is not an alias") cmd end end #handle clean methods def resolve_hooks(cmd, macrostep, microstep) if (cmd.key =~ /on_(.*)clean/ || cmd.key =~ /on_(.*)init/) cmds = [] if cmd.value.kind_of?(Array) cmds = cmd.value.map do |cmd| resolve_alias(cmd) end cmds.flatten! else fail RecipeError, "Invalid #{cmd.key} arguments" end if cmd.key.eql? "on_clean" microstep_name = "_clean_#{macrostep.clean_microsteps.count}" \ "_#{microstep.name}" new_clean_microstep = Microstep.new({microstep_name => []}) new_clean_microstep.on_checkpoint = microstep.on_checkpoint new_clean_microstep.commands = cmds.clone macrostep.clean_microsteps.unshift new_clean_microstep return elsif cmd.key.eql? "on_init" microstep_name = "_init_#{macrostep.init_microsteps.count}"\ "_#{microstep.name}" new_init_microstep = Microstep.new({microstep_name=> []}, microstep) new_init_microstep.on_checkpoint = microstep.on_checkpoint new_init_microstep.commands = cmds.clone macrostep.init_microsteps.unshift new_init_microstep return else @sections.values.each do |section| section.clean_macrostep if cmd.key.eql? "on_#{section.name}_clean" microstep_name = "_clean_#{section.clean_macrostep.microsteps.count}" \ "_#{microstep.name}" new_clean_microstep = Microstep.new({microstep_name=> []}) new_clean_microstep.commands = cmds.clone new_clean_microstep.on_checkpoint = microstep.on_checkpoint section.clean_macrostep.microsteps.unshift new_clean_microstep return elsif cmd.key.eql? "on_#{section.name}_init" microstep_name = "_init_#{section.init_macrostep.microsteps.count}" \ "_#{microstep.name}" new_init_microstep = Microstep.new({microstep_name=> []}) new_init_microstep.commands = cmds.clone new_init_microstep.on_checkpoint = microstep.on_checkpoint section.init_macrostep.microsteps.push new_init_microstep return end end end fail RecipeError, "Invalid command: '#{cmd.key}'" else return cmd end end def microsteps if @microsteps.nil? microsteps = [] @sections.values.each do |section| section.sequence do |macrostep| macrostep.sequence do |microstep| microsteps.push microstep end end end @microsteps = microsteps end return @microsteps end def all_checkpoints if @all_checkpoints.nil? @all_checkpoints = [] microsteps.each do |m| step = m.slug while @all_checkpoints.map{|c| c["step"]}.include?(step) do prefix, suffix = step.split("%") step = if suffix.nil? "#{prefix}%1" else "#{prefix}%#{suffix.to_i+1}" end end @all_checkpoints.push({ 'id' => m.identifier, 'step' => step }) end end return @all_checkpoints end def calculate_step_identifiers Kameleon.ui.debug("Calculating microstep identifiers") base_salt = "" order = 0 @sections.values.each do |section| section.sequence do |macrostep| macrostep.sequence do |microstep| if ["redo", "skip"].include? microstep.on_checkpoint microstep.calculate_identifier "" else base_salt = microstep.calculate_identifier base_salt end slug = "#{section.name}/#{macrostep.name}/#{microstep.name}" microstep.slug = slug microstep.order = (order += 1) Kameleon.ui.debug(" #{microstep.slug}: #{microstep.identifier}") end end end end def flatten_data files = [] @data_files.each do |d| if d.directory? Find.find("#{d}") do |f| files.push(Pathname.new(f)) unless File.directory? f end else files.push(d) end end @data_files = files.uniq end def to_hash recipe_hash = { "name" => @name, "path" => @path.to_s, "base_recipes_files" => @base_recipes_files.map {|p| p.to_s }, "step_files" => @step_files.map {|p| p.to_s }, "env_files" => @env_files.map {|p| p.to_s }, "data_files" => @data_files.map {|p| p.to_s }, "global" => @global, "aliases" => @aliases, } recipe_hash["checkpoint"] = @checkpoint unless @checkpoint.nil? recipe_hash["steps"] = to_array return recipe_hash end def display_info(do_relative_path) def prefix Kameleon.ui.shell.say " -> ", :magenta end def relative_or_absolute_path(do_relative_path, path) if do_relative_path return path.relative_path_from(Pathname(Dir.pwd)) else return path end end Kameleon.ui.shell.say "--------------------" Kameleon.ui.shell.say "[Name]", :red prefix ; Kameleon.ui.shell.say "#{@name}" Kameleon.ui.shell.say "[Path]", :red prefix ; Kameleon.ui.shell.say relative_or_absolute_path(do_relative_path, @path), :cyan Kameleon.ui.shell.say "[Description]", :red prefix ; Kameleon.ui.shell.say "#{@metainfo['description']}" Kameleon.ui.shell.say "[Parent recipes]", :red (@base_recipes_files - [@path]).each do |base_recipe_file| prefix ; Kameleon.ui.shell.say relative_or_absolute_path(do_relative_path, base_recipe_file), :cyan end Kameleon.ui.shell.say "[Steps]", :red @step_files.each do |step| prefix ; Kameleon.ui.shell.say relative_or_absolute_path(do_relative_path, step), :cyan end Kameleon.ui.shell.say "[Data]", :red @data_files.each do |d| prefix ; Kameleon.ui.shell.say relative_or_absolute_path(do_relative_path, d), :cyan end Kameleon.ui.shell.say "[Environment scripts]", :red @env_files.each do |d| prefix ; Kameleon.ui.shell.say relative_or_absolute_path(do_relative_path, d), :cyan end Kameleon.ui.shell.say "[Variables]", :red @global.sort.map do |key, value| value = "\n" if value.to_s.empty? prefix ; Kameleon.ui.shell.say "#{key}: ", :yellow Kameleon.ui.shell.say "#{value}" end end def to_array array = [] @sections.values.each do |section| section.to_array.each { |m| array.push m } end return array end def all_files return @base_recipes_files + @step_files + @data_files + @env_files end end class RecipeTemplate < Recipe def initialize(path, kwargs = {}) super(path, kwargs) end def relative_path_from_recipe(recipe_path) recipe_path = Pathname.new(recipe_path) relative_path_tpl_repo = @path.relative_path_from(Kameleon.env.repositories_path) absolute_path = Pathname.new(Kameleon.env.workspace).join(relative_path_tpl_repo) return absolute_path.relative_path_from(recipe_path.dirname) end end end kameleon-builder-2.11.0/tests/0000755000004100000410000000000015005702337016206 5ustar www-datawww-datakameleon-builder-2.11.0/tests/issue76/0000755000004100000410000000000015005702337017513 5ustar www-datawww-datakameleon-builder-2.11.0/tests/issue76/steps/0000755000004100000410000000000015005702337020651 5ustar www-datawww-datakameleon-builder-2.11.0/tests/issue76/steps/testbootstrap.yaml0000644000004100000410000000011515005702337024447 0ustar www-datawww-data- toto: $${arch}-toto - step: - exec_local: echo $${iso_filename} $${toto} kameleon-builder-2.11.0/tests/issue76/fail.yaml0000644000004100000410000000032615005702337021313 0ustar www-datawww-dataglobal: arch: x86_64 iso_arch: $${arch} iso_filename: debian-jessie-$${iso_arch}-live.iso bootstrap: - testbootstrap: - toto: $${arch} - step: - exec_local: echo $iso_filename kameleon-builder-2.11.0/tests/issue76/ok.stdout0000644000004100000410000000214515005702337021372 0ustar www-datawww-dataStarting recipe consistency check Resolving variables Warning : variable $$iso_filename is not enclosed with braces, which may cause errors. Please prefer using $${iso_filename}. Warning : variable $$toto is not enclosed with braces, which may cause errors. Please prefer using $${toto}. Warning : variable $$titi is not enclosed with braces, which may cause errors. Please prefer using $${titi}. Creating kameleon build directory : /home/neyron/scm/OAR/kameleon/tests/issue76/build/ok Starting build recipe 'ok.yaml' Step _init_bootstrap took: 0 secs Step 1 : bootstrap/testbootstrap/step --> Running the step... Starting command: "bash" [local] The local_context has been initialized [local] debian-jessie-x86_64-live.iso x86_64-toto Step testbootstrap took: 1 secs Step 2 : bootstrap/testbootstrapinline/step --> Running the step... [local] debian-jessie-x86_64-live.iso Step testbootstrapinline took: 0 secs Step _clean_bootstrap took: 0 secs Step _init_setup took: 0 secs Step _clean_setup took: 0 secs Step _init_export took: 0 secs Step _clean_export took: 0 secs Successfully built 'ok.yaml' Total duration : 1 secs kameleon-builder-2.11.0/tests/issue76/fail.stdout0000644000004100000410000000052515005702337021674 0ustar www-datawww-dataWarning : variable $$arch is not enclosed with braces, which may cause errors. Please prefer using $${arch}. Warning : variable $$arch-live is not enclosed with braces, which may cause errors. Please prefer using $${arch-live}. Error : /home/neyron/scm/OAR/kameleon/tests/issue76/fail.yaml: variable $$arch-live not found in local or global kameleon-builder-2.11.0/tests/issue76/ok.yaml0000644000004100000410000000037115005702337021011 0ustar www-datawww-dataglobal: arch: x86_64 iso_arch: $${arch} iso_filename: debian-jessie-$${iso_arch}-live.iso bootstrap: - testbootstrap - testbootstrapinline: - titi: $arch - step: - exec_local: echo $${iso_filename} $${titi} kameleon-builder-2.11.0/tests/test_version.rb0000644000004100000410000000021615005702337021256 0ustar www-datawww-datarequire File.expand_path("../helper", __FILE__) describe Kameleon do it "must be defined" do Kameleon::VERSION.wont_be_nil end end kameleon-builder-2.11.0/tests/helper.rb0000644000004100000410000000070415005702337020013 0ustar www-datawww-dataif ENV['COVERAGE'] require 'simplecov' require 'coveralls' SimpleCov.profiles.define 'kameleon' do add_filter './tests/' add_filter './spec/' add_filter './autotest/' add_group 'Binaries', 'bin/' add_group 'Libraries', 'lib/' end SimpleCov.start 'kameleon' SimpleCov.merge_timeout 300 SimpleCov.command_name 'unit' end require 'minitest/unit' require 'minitest/autorun' require 'minitest/pride' require 'kameleon' kameleon-builder-2.11.0/tests/recipes/0000755000004100000410000000000015005702337017640 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/0000755000004100000410000000000015005702337020776 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/checkpoints/0000755000004100000410000000000015005702337023310 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/checkpoints/test.yaml0000644000004100000410000000061515005702337025155 0ustar www-datawww-dataenabled?: - exec_local: touch $KAMELEON_WORKDIR/list_checkpoint.txt create: - exec_local: echo @microstep_id | tee -a $KAMELEON_WORKDIR/list_checkpoint.txt apply: - exec_local: echo @microstep_id list: - exec_local: touch $KAMELEON_WORKDIR/list_checkpoint.txt - exec_local: cat $KAMELEON_WORKDIR/list_checkpoint.txt clear: - exec_local: rm -f $KAMELEON_WORKDIR/list_checkpoint.txt kameleon-builder-2.11.0/tests/recipes/steps/test_uuid_step.yaml0000644000004100000410000000037615005702337024730 0ustar www-datawww-data- my_uuid: $${kameleon_short_uuid} - TEST_VAR: toto_tata - version: 12.2 - variant: toto-tata - my_distrib: $${version}-$${variant} - do_bug: - exec_local: | echo my_uuid: $${my_uuid} echo TEST_VAR: $${TEST_VAR} echo $${my_distrib} kameleon-builder-2.11.0/tests/recipes/steps/setup/0000755000004100000410000000000015005702337022136 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/setup/linux/0000755000004100000410000000000015005702337023275 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/setup/linux/software_install.yaml0000644000004100000410000000021215005702337027534 0ustar www-datawww-data# # Dummy Software Install - intall: - exec_in: echo install - configure: - exec_out: echo configure - clean: - exec_in: echo clean kameleon-builder-2.11.0/tests/recipes/steps/aliases/0000755000004100000410000000000015005702337022417 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/aliases/defaults.yaml0000644000004100000410000000412115005702337025110 0ustar www-datawww-datawrite_local: - exec_local: | mkdir -p $(dirname @1); cat >@1 <@1 <@1 <>@1 <>@1 <>@1 < @2 local2in: - exec_in: mkdir -p $(dirname @2) - pipe: - exec_local: cat @1 - exec_in: cat > @2 out2local: - exec_local: mkdir -p $(dirname @2) - pipe: - exec_out: cat @1 - exec_local: cat > @2 out2in: - exec_in: mkdir -p $(dirname @2) - pipe: - exec_out: cat @1 - exec_in: cat > @2 in2local: - exec_local: mkdir -p $(dirname @2) - pipe: - exec_in: cat @1 - exec_local: cat > @2 in2out: - exec_out: mkdir -p $(dirname @2) - pipe: - exec_in: cat @1 - exec_out: cat > @2 check_cmd_out: - rescue: - exec_out: command -V @1 2> /dev/null - breakpoint: "@1 is missing from out_context" check_cmd_local: - rescue: - exec_local: command -V @1 2> /dev/null - breakpoint: "@1 is missing from local_context" check_cmd_in: - rescue: - exec_in: command -V @1 2> /dev/null - breakpoint: "@1 is missing from in_context" umount_out: - exec_out: | echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true umount_local: - exec_local: | echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true umount_in: - exec_in: | echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true download_file_in: - exec_in: __download "@1" "@2" download_file_out: - exec_out: __download "@1" "@2" download_file_local: - exec_local: __download "@1" "@2" kameleon-builder-2.11.0/tests/recipes/steps/bootstrap/0000755000004100000410000000000015005702337023013 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/bootstrap/linux/0000755000004100000410000000000015005702337024152 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/bootstrap/linux/bootstrap.yaml0000644000004100000410000000005515005702337027053 0ustar www-datawww-data- bootstrap: - exec_out: echo $include_pkg kameleon-builder-2.11.0/tests/recipes/steps/bootstrap/BSD/0000755000004100000410000000000015005702337023423 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/bootstrap/BSD/bootstrap.yaml0000644000004100000410000000010715005702337026322 0ustar www-datawww-data- bootstrap: - exec_local: echo BSD - exec_out: echo $include_pkg kameleon-builder-2.11.0/tests/recipes/steps/export/0000755000004100000410000000000015005702337022317 5ustar www-datawww-datakameleon-builder-2.11.0/tests/recipes/steps/export/save_appliance.yaml0000644000004100000410000000036415005702337026160 0ustar www-datawww-data# # Dummy Save Appliance - save_as_raw: - exec_local: echo save_as_raw - exec_out: echo save_as_raw - exec_in: echo save_as_raw - save_as_tgz: - exec_local: echo save_as_tgz - exec_out: echo save_as_tgz - exec_in: echo save_as_tgz kameleon-builder-2.11.0/tests/recipes/steps/local_variables.yaml0000644000004100000410000000017415005702337025006 0ustar www-datawww-data- my_distrib: $${version}-$${variant} - show_variables: - exec_local: "echo This is the distrib variable: $${my_distrib}" kameleon-builder-2.11.0/tests/recipes/steps/enable_something.yaml0000644000004100000410000000007315005702337025165 0ustar www-datawww-data- enable_something: - exec_local: echo enable_something kameleon-builder-2.11.0/tests/recipes/test_recipe_checkpoints.yaml0000644000004100000410000000341215005702337025424 0ustar www-datawww-data#=============================================================================== # vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80 #=============================================================================== # # DESCRIPTION: This is a test recipe made for unit testing # #=============================================================================== --- # Loads some helpful aliases extend: ../test2/test2.yaml aliases: defaults.yaml checkpoint: test.yaml #== Global variables use by Kameleon engine and the steps global: include_steps: - $${distrib} ## User varibales : used by the recipe user_name: kameleon_user user_password: $${user_name} # test overload toto: $${toto} tata # Distribution distrib: linux ## System variables. Required by kameleon engine # Include specific steps include_steps: - $${distrib} bootstrap_packages: > less vim python sl sudo version: 12.2 variant: toto-tata uuid: $${kameleon_uuid} appliance_filename: $${kameleon_recipe_name}_$${kameleon_short_uuid} #== Bootstrap the new system and create the 'in_context' bootstrap: setup: - MA: - MA1: - on_checkpoint: redo - exec_local: echo "MicroStepA1 ($${checkpointing_enabled}, $${persistent_cache})" - MB: - MB1: - exec_local: echo MicroStepB1; cp; echo coucou - MB2: - on_checkpoint: skip - exec_local: echo MicroStepB2 - MB3: - exec_local: echo MicroStepB3 - MC: - MC1: - exec_local: echo MicroStepC1 - MC1: - exec_local: echo MicroStepC1 - MC2: - on_checkpoint: only - exec_local: echo MicroStepC2 - MC3: - exec_local: echo MicroStepC3 - MD: - MD1: - on_checkpoint: use_cache - exec_local: echo MicroStepD1 export: kameleon-builder-2.11.0/tests/recipes/test_recipe.yaml0000644000004100000410000000330415005702337023032 0ustar www-datawww-data#=============================================================================== # vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80 #=============================================================================== # # DESCRIPTION: This is a test recipe made for unit testing # #=============================================================================== --- # Loads some helpful aliases extend: ../test2/test2.yaml aliases: defaults.yaml checkpoint: test.yaml #== Global variables use by Kameleon engine and the steps global: include_steps: - $${distrib} ## User varibales : used by the recipe user_name: kameleon_user user_password: $${user_name} # test overload toto: $${toto} tata # Distribution distrib: linux ## System variables. Required by kameleon engine # Include specific steps include_steps: - $${distrib} bootstrap_packages: > less vim python sl sudo version: 12.2 variant: toto-tata uuid: $${kameleon_uuid} appliance_filename: $${kameleon_recipe_name}_$${kameleon_short_uuid} #== Bootstrap the new system and create the 'in_context' bootstrap: - enable_something - bootstrap: - include_pkg: $${bootstrap_packages} setup: - software_install - inline_step: - do_something: - exec_local: > echo $${toto} ;\ echo titi - do_something_else: - exec_local: | echo $${toto} $${user_name} - test_data: - exec_local: cat $${kameleon_data_dir}/mydata.txt - exec_local: cat $${kameleon_data_dir}/mydata.txt - local_variables - test_uuid_step: - my_uuid: $${kameleon_uuid}_$${kameleon_recipe_name} export: - save_appliance: - save_as_raw # - save_as_vmdk kameleon-builder-2.11.0/tests/recipes/test_recipe_child.yaml0000644000004100000410000000033615005702337024177 0ustar www-datawww-dataextend: test_recipe global: mo: mi test_override: $${user_name}-override setup: - inline_step: - do_something_else: - exec_local: | echo $${toto} $${user_name} echo $${test_override} kameleon-builder-2.11.0/tests/test_context.rb0000644000004100000410000000060015005702337021252 0ustar www-datawww-datarequire File.expand_path("../helper", __FILE__) class TestLocalContext < Minitest::Unit::TestCase def setup @local = Kameleon::LocalContext.new end def test_echo_cmd out = capture_io{ @local.exec "echo mymessage" }.join '' assert_equal "mymessage\n", out out = capture_io{ @local.exec "echo >&2 myerror" }.join '' assert_equal "myerror\n", out end end kameleon-builder-2.11.0/tests/test2/0000755000004100000410000000000015005702337017247 5ustar www-datawww-datakameleon-builder-2.11.0/tests/test2/steps/0000755000004100000410000000000015005702337020405 5ustar www-datawww-datakameleon-builder-2.11.0/tests/test2/steps/data/0000755000004100000410000000000015005702337021316 5ustar www-datawww-datakameleon-builder-2.11.0/tests/test2/steps/data/mydata.txt0000644000004100000410000000003215005702337023331 0ustar www-datawww-dataThis is OVERRIDED DATA!!! kameleon-builder-2.11.0/tests/test2/test3.yaml0000644000004100000410000000002515005702337021172 0ustar www-datawww-dataglobal: toto: titi kameleon-builder-2.11.0/tests/test2/test2.yaml0000644000004100000410000000010115005702337021164 0ustar www-datawww-dataextend: ./test3.yaml global: toto: $${toto} test $${user_name} kameleon-builder-2.11.0/tests/test2/test_data.yaml0000644000004100000410000000021715005702337022103 0ustar www-datawww-dataextend: ../recipes/test_recipe.yaml setup: - "@base" - test_data: - do_test: - exec_local: cat $${kameleon_data_dir}/mydata.txt kameleon-builder-2.11.0/tests/test_recipe.rb0000644000004100000410000000051315005702337021040 0ustar www-datawww-datarequire File.expand_path("../helper", __FILE__) class TestRecipe < Minitest::Unit::TestCase def setup Kameleon.ui.level = "silent" @recipe = Kameleon::Recipe.new File.join(File.dirname(__FILE__), "recipes/dummy_recipe.yaml") end def test_dummy_recipe_name assert_equal @recipe.name, "dummy_recipe" end end kameleon-builder-2.11.0/RELEASING.md0000644000004100000410000000331115005702337016675 0ustar www-datawww-dataAbout releases: =============== You can use the script ``./scripts/bumpversion.py`` which will handle everything (incrementation, git tag creation, changelog update). First you need to install bumpversion using pip:: ``` sudo pip install bumpversion ``` Assuming work is done in the devel branch. For stable releases: -------------------- ## Switch and update the master branch ``` git switch master git pull -r ``` Do whatever changes needed in the code (possibly merging a branch or a pull request). Commit. ## Update changelog Edit the `CHANGES` file. Commit. ## Bump version Edit the `lib/kameleon/version.rb` file and bump the version. Commit: ``` git commit -m "v2.10.16 → v.2.10.17" lib/kameleon/version.rb ``` ## Build gem ``` gem build kameleon-builder.gemspec ``` ## Test Manually install: ``` gem install --user ./kameleon-builder-2.10.17.gem ``` Test, test, test. ## Tag If everything is ok, tag: ``` git tag -s 'v2.10.17' -m 'v2.10.17' ``` ## Push git push ``` git push ``` ## Push to Ruby gem repository ``` gem push kameleon-builder-2.10.17.gem ``` Note: You need a rubygem account and the owner has to give you permissions so that you can push. To do so, create an account on https://rubygems.org/ and ask an owner to do the following command:: ``` gem owner kameleon-builder -a your@email.com ``` That's all :) ## In case a release is buggy Yank it to remove it from the rubygems index. ``` gem yank kameleon-builder -v 2.10.16 ``` For developments: ----------------- Changes can be tested by locally installing the gem after building it: ``` gem build kameleon-builder.gemspec && gem install --user ./kameleon-builder-2.10.17.gem ``` Using git branches, github pull requests, aso, is of course good. kameleon-builder-2.11.0/completion/0000755000004100000410000000000015005702337017215 5ustar www-datawww-datakameleon-builder-2.11.0/completion/kameleon.bash0000644000004100000410000000147215005702337021653 0ustar www-datawww-data_kameleon() { COMPREPLY=() local i=1 while [ $i -le "$COMP_CWORD" -a "${COMP_WORDS[$i]:0:1}" == "-" ]; do ((i++)) done if [ "$COMP_CWORD" -eq $i ]; then local commands="$(compgen -W "$(kameleon commands)" -- "${COMP_WORDS[$i]}")" COMPREPLY=( $commands $projects ) fi while [ $i -le "$COMP_CWORD" -a "${COMP_WORDS[$i]:0:1}" == "-" ]; do ((i++)) done if [ "$COMP_CWORD" -eq $((i+1)) ] && kameleon commands | grep -q "${COMP_WORDS[$i]}" ; then if kameleon help | grep -qe "^ kameleon ${COMP_WORDS[$i]}[a-z]* "; then local commands="$(compgen -W "$(kameleon ${COMP_WORDS[$i]} commands)" -- "${COMP_WORDS[$((i+1))]}")" COMPREPLY=( $commands $projects ) fi fi } complete -o default -F _kameleon kameleon kameleon-builder-2.11.0/completion/_kameleon0000644000004100000410000001506315005702337021077 0ustar www-datawww-data#compdef kameleon # This is ZSH completion script based on the kameleon 2.6.0.dev API # # Author: Michael Mercier # # This was made with the help of this howto: # https://github.com/zsh-users/zsh-completions/blob/master/zsh-completions-howto.org # and with the _git completion file # Helpers template_list="kameleon template list | tail -n +4 | cut -f1 -d'|'" # Repository kameleon subcommands _kameleon_repository () { local curcontext=$curcontext state line ret=1 declare -A opt_args _arguments -C \ ': :->command' \ '*:: :->option-or-argument' && ret=0 case $state in (command) declare -a commands commands=( add:'Adds a new named NAME repository at URL' help:'Describe available subcommands or one specific subcommand' list:'Lists available repositories' update:'Update specified repository') _describe -t commands command commands && ret=0 ;; (option-or-argument) curcontext=${curcontext%:*}-$line[1]: case $line[1] in (add) _arguments \ '1:Repository local name' \ '2:Repository URL or local path:_urls' \ '(-b --branch=)'{-b,--branch=}':To checkout BRANCH' && ret=0 ;; (help) _arguments \ '::Repository sub command:(add list update)' && ret=0 ;; (update) _arguments \ ":Update the repository named NAME:($(kameleon repository list))" && ret=0 ;; esac esac return ret } # Template kameleon subcommands _kameleon_template () { local curcontext=$curcontext state line ret=1 declare -A opt_args _arguments -C \ ': :->command' \ '*:: :->option-or-argument' && ret=0 case $state in (command) declare -a commands commands=( help:'Describe available subcommands or one specific subcommand' import:'Imports the given template' info:'Display detailed information about a template' list:'Lists available templates' repository:'alias for \"kameleon repository\"') _describe -t commands command commands && ret=0 ;; (option-or-argument) curcontext=${curcontext%:*}-$line[1]: case $line[1] in (help) _arguments \ '::Template sub command:(import info list repository)' && ret=0 ;; (import) _arguments \ ":template name:($(kameleon template list | tail -n +4 | cut -f1 -d'|'))" && ret=0 ;; (info) _arguments \ ":template name:($(kameleon template list | tail -n +4 | cut -f1 -d'|'))" && ret=0 '*'{-g,--global}':Custom global variable define as key\:value' && ret=0 ;; (repository) _kameleon_repository && ret=0 ;; esac esac return ret } # TODO: # - add checkpoint list completion # - add key completion for global # - factorize some options # - allow common options everywere (not just befor command) _kameleon () { local curcontext=$curcontext state line ret=1 declare -A opt_args _arguments -C \ '(--no-color)--color[Enables colorization in output (Default)]' \ '(--color)--no-color[Disables colorization in output]' \ '(--no-verbose --debug)--verbose[Enables verbose output for kameleon Users]' \ '(--verbose)--no-verbose[Disables verbose output for kameleon Users (Default)]' \ '(--no-debug)--debug[Enables debug output for kameleon Developpers]' \ '(--debug --verbose)--no-debug[Disables debug output for kameleon Users and Developpers(Default)]' \ '(--no-script)--script[Never prompts for User intervention]' \ '(--script)--no-script[Prompts for user intervention if necessary (Default)]' \ ': :->command' \ '*:: :->option-or-argument' && ret=0 case $state in (command) declare -a commands commands=( build:'Builds the appliance from the given recipe' help:'Describe available commands or one specific command' info:'Display detailed information about a recipe' list:'Lists all defined recipes in the current directory' new:'Creates a new recipe' repository:'Manages set of remote git repositories' template:'Lists and imports templates' version:'Prints the Kameleon version information') _describe -t commands command commands && ret=0 ;; (option-or-argument) curcontext=${curcontext%:*}-$line[1]: case $line[1] in (build) _arguments -C \ '(--from-cache)1:Kameleon recipe path:_files' \ '(1)--from-cache=:Persistent cache tar file to build the image:_files' \ '--cache-archive-compression=:Select the persistent cache compression:(gzip xz bz2)' \ '(-b --build-path)'{-b,--build-path=}':Build directory path:_directories' \ '--clean[Runs the command `kameleon clean` first]' \ '--from-checkpoint=:Specify checkpoint to build the image' \ '--enable-checkpoint[Enables checkpoint \[experimental\]]' \ {--checkpoints,--list-checkpoints}'[Lists all availables checkpoints]' \ '--enable-cache[Generates a persistent cache for the appliance]' \ '--cache-path=:Cache directory path:_directories' \ '--proxy=:Specifies the hostname and port number of the parent HTTP proxy define as host\:port' \ '--proxy-credentials=:Specifies the username and password if the parent proxy requires authorisation define as username\:password' \ '*'{-g,--global}':Custom global variable define as key\:value' && ret=0 ;; (help) _arguments \ ":Kameleon commands:($(kameleon commands))" && ret=0 ;; (info) _arguments \ '*'{-g,--global}':Custom global variable define as key\:value' \ '(--from-cache)1:Kameleon recipe path:_files' \ '(1)--from-cache=:Persistent cache tar file to get info from:_files' && ret=0 ;; (list) _nothing ;; (new) _arguments \ '1:Kameleon recipe path:_files' \ '2:Template name:->template_name' && ret=0 case "$state" in template_name) _alternative ":templates name:($(kameleon template list | tail -n +4 | cut -f1 -d'|'))" && ret=0 ;; esac ;; (repository) _kameleon_repository && ret=0 ;; (template) _kameleon_template && ret=0 ;; (*) _nothing ;; esac ;; esac return ret } _kameleon "$@" kameleon-builder-2.11.0/completion/kameleon.fish0000644000004100000410000000135615005702337021670 0ustar www-datawww-datacomplete -c kameleon -l help --description "Describe available commands or one specific command" \ -a 'help new templates version build checkpoints clear' complete -c kameleon -l build --description "Builds the appliance from the recipe" complete -c kameleon -l clean --description "Cleaning 'out' and 'local' context and removing all checkpoints" complete -c kameleon -l checkpoints --description "Lists all availables checkpoints" complete -c kameleon -l version --description "Prints the Kameleon's version information" complete -c kameleon -l new --description "Creates a new recipe" complete -c kameleon -l import --description "Imports the given template" complete -c kameleon -l templates --description "Lists all defined templates" kameleon-builder-2.11.0/Gemfile0000644000004100000410000000071515005702337016342 0ustar www-datawww-datasource "https://rubygems.org" gem "syck", :platforms => [:ruby_20, :ruby_21] group :development do gem "pry" gem "rake" platforms :ruby_19, :ruby_20 do gem "pry-debugger" gem "pry-stack_explorer" end end group :test do gem "coveralls", ">= 0.5.7", :require => false gem "rspec", ">= 3" gem "rspec-mocks", ">= 3" gem "rubocop", ">= 0.19", :platforms => [:ruby_19, :ruby_20, :ruby_21] gem "simplecov", :require => false end gemspec kameleon-builder-2.11.0/COPYING0000644000004100000410000004312515005702337016104 0ustar www-datawww-data GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. kameleon-builder-2.11.0/lib640000777000004100000410000000000015005702337016532 2./libustar www-datawww-datakameleon-builder-2.11.0/Vagrantfile0000644000004100000410000000116315005702337017232 0ustar www-datawww-data# -*- mode: ruby -*- # vi: set ft=ruby : # Require vagrant-kvm plugin ENV['VAGRANT_DEFAULT_PROVIDER'] = 'kvm' Vagrant.configure("2") do |config| config.vm.box = "oar-team/kameleon-dev" config.vm.hostname = "kameleon-devel" config.vm.provision "docker", images: ["scratch"] config.ssh.forward_x11 = true config.ssh.forward_agent = true config.vm.provider :kvm do |kvm| kvm.memory_size = "2GiB" kvm.core_number = 2 end config.vm.synced_folder ".", "/home/vagrant/kameleon" config.vm.provision "shell", privileged: false, inline: <<-EOF cd /home/vagrant/kameleon && bundle install EOF end kameleon-builder-2.11.0/erb/0000755000004100000410000000000015005702337015614 5ustar www-datawww-datakameleon-builder-2.11.0/erb/userconf.yaml.erb0000644000004100000410000000043015005702337021070 0ustar www-datawww-data--- repositories_path: <%= Kameleon.default_values[:repositories_path] %> extend_yaml_erb: <%= Kameleon.default_values[:extend_yaml_erb] %> script: <%= Kameleon.default_values[:script] %> color: <%= Kameleon.default_values[:color] %> debug: <%= Kameleon.default_values[:debug] %> kameleon-builder-2.11.0/erb/extend.yaml.erb0000644000004100000410000000130415005702337020534 0ustar www-datawww-data#============================================================================== # vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80 #============================================================================== # # DESCRIPTION: # #============================================================================== # This recipe extends another. To look at the step involed, run: # kameleon build --dryrun <%= recipe_name %> # To see the variables that you can override, use the following command: # kameleon info <%= recipe_name %> --- extend: <%= tpl.relative_path_from_recipe(recipe_path) %> global: bootstrap: - "@base" setup: - "@base" export: - "@base" kameleon-builder-2.11.0/.env0000644000004100000410000000557115005702337015645 0ustar www-datawww-data#!/usr/bin/env bash export QEMU_AUDIO_DRV=alsa export GIT_PROJECT=kameleon export ROOT_PROJECT=$(dirname $(readlink -f ${BASH_SOURCE[0]})) export KAMELEON_LOG=${KAMELEON_LOG:-"info"} # enable kameleon autocompletion source $ROOT_PROJECT/completion/kameleon.bash QEMU_MONITOR_PORT=11023 function kameleon { BUNDLE_GEMFILE=$ROOT_PROJECT/Gemfile bundle exec kameleon "$@" } function devrun_chroot_clean() { mount \ | grep rootfs | grep kameleon \ | py "s='rootfs'" "l=x.split(' ')[2].split(s)[0]" "print(l)" \ | uniq \ | xargs -I {} sudo bash "$ROOT_PROJECT"/contrib/scripts/umount-chroot.sh -p -f -a -k -y -c "{}" } function devrun_clear() { KAMELEON_WORKDIR=${KAMELEON_WORKDIR:-"/tmp/kameleon/"} RECIPE_DEV_NAME=${RECIPE_DEV_NAME:-"mymachine"} devrun_chroot_clean cd $KAMELEON_WORKDIR kameleon clear $RECIPE_DEV_NAME.yaml if [ -d "$KAMELEON_WORKDIR/build" ]; then sudo rm "$KAMELEON_WORKDIR/build" -fr 2> /dev/null fi } function devrun_qemu_stop() { if nc -w 0 -z localhost $QEMU_MONITOR_PORT 2>/dev/null then echo "Shutting down qemu virtual machine" echo "system_reset" | nc localhost $QEMU_MONITOR_PORT echo "" fi while nc -w 0 -z localhost $QEMU_MONITOR_PORT do sleep 0.5 echo -n "." done echo " ~> OK" } function devrun_qemu() { KAMELEON_WORKDIR=${KAMELEON_WORKDIR:-"/tmp/kameleon/"} RECIPE_DEV_NAME=${RECIPE_DEV_NAME:-"mymachine"} DRIVE="$KAMELEON_WORKDIR/build/$RECIPE_DEV_NAME/${RECIPE_DEV_NAME}.qcow2" RAM_COUNT=2048 SPICE_PORT=5900 CPU_COUNT=2 sudo qemu-system-x86_64 \ -name "$VM_NAME" \ -machine pc-1.2,accel=kvm,usb=off -m 2048 -realtime mlock=off \ -smp $CPU_COUNT \ -drive file="$DRIVE",cache=none,media=disk,if=virtio,id=drive0 \ -m $RAM_COUNT \ -no-reboot \ -rtc base=utc \ -soundhw ac97 \ -net nic,model=virtio -net user \ -usb \ -usbdevice tablet \ -vga qxl \ -spice port=$SPICE_PORT,addr=127.0.0.1,disable-ticketing,image-compression=auto_lz \ -device virtio-serial-pci \ -device virtserialport,chardev=spicechannel0,name=com.redhat.spice.0 \ -chardev spicevmc,id=spicechannel0,name=vdagent \ -monitor tcp::$QEMU_MONITOR_PORT,server,nowait \ -daemonize spicec -h 127.0.0.1 -p $SPICE_PORT --title "$VM_NAME" || : devrun_qemu_stop } function devrun_build() { KAMELEON_WORKDIR=${KAMELEON_WORKDIR:-"/tmp/kameleon/"} RECIPE_DEV_NAME=${RECIPE_DEV_NAME:-"mymachine"} TEMPLATE=${1:-"chroot/debian7"} mkdir -p $KAMELEON_WORKDIR && cd $KAMELEON_WORKDIR cmd="kameleon new $RECIPE_DEV_NAME $TEMPLATE \ && kameleon build $RECIPE_DEV_NAME.yaml" eval "$cmd" if [ -d "$KAMELEON_WORKDIR/build/$RECIPE_DEV_NAME" ]; then cd "$KAMELEON_WORKDIR/build/$RECIPE_DEV_NAME" fi }