diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..6b2411b4de447d50510d630dcd7e92860816f883 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# Byte-compiled & optimised +__pycache__/ +*.py[cod] +*$py.class + +# Eclipse +.project +.pydevproject +.settings + +# OSX +.DS_Store + +# Data +data* + diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 456777a81f336c1455622700bff482275ffef386..c27286cc496cc27b6992174dbc5558a70b3fd1dc --- a/README.md +++ b/README.md @@ -1,2 +1,39 @@ -# Rosetta -A container-centric Science Platform +# Rosetta 💁🏽 + + +_A container-centric Science Platform_ + + +Rosetta makes it easy to run interactive workloads on batch and remote computing systems using Docker and Singularity containers. + + +## Quickstart + +Requirements: + + Bash, Git and Docker. Runs on Linux, Mac or Windows*. + +*Windows not supported in development mode due to lack of support for symbolic links. + +Setup + + $ rosetta/setup + +Build + + $ rosetta/build + +Run + + $ rosetta/run + +Check status + + $ rosetta/ps + + +### Building errors + +It is common for the build process to fail with a "404 not found" error on an apt-get instrucions, as apt repositories often change their IP addresses. In such case, try: + + $ rosetta/build nocache \ No newline at end of file diff --git a/Software/MetaDesktop/Dockerfile b/Software/MetaDesktop/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ea43e98371c8cb7d3a6d40ec56f1cf80152d7250 --- /dev/null +++ b/Software/MetaDesktop/Dockerfile @@ -0,0 +1,107 @@ +FROM ubuntu:18.04 +MAINTAINER Stefano Alberto Russo <stefano.russo@gmail.com> + +#---------------------- +# Basics +#---------------------- + +# Set non-interactive +ENV DEBIAN_FRONTEND noninteractive + +# Update first of all +RUN apt-get update + +# Utilities +RUN apt-get install -y nano telnet unzip wget supervisor build-essential python-dev git-core openjdk-8-jre + + +#------------------------ +# "Meta" user +#------------------------ + +# Add group. We chose GID 65527 to try avoiding conflicts. +RUN groupadd -g 65527 metauser + +# Add user. We chose UID 65527 to try avoiding conflicts. +RUN useradd metauser -d /metauser -u 65527 -g 65527 -m -s /bin/bash + +# Add metuaser user to sudoers +RUN adduser metauser sudo + +# Install suodo +RUN apt-get install sudo -y + +# No pass sudo (for everyone, actually) +COPY files/sudoers /etc/sudoers + +#------------------------ +# Supervisor conf +#------------------------ + +# Supervisord conf +COPY files/supervisord.conf /etc/supervisor/ + +#------------------------ +# VNC +#------------------------ + +# Install xvfb that triggers minimal install of X base packages and xterm as sample application +RUN apt-get install xvfb xterm -y + +# Install base packages for VNC server and headless desktop (2) +RUN cd /opt && wget https://bintray.com/tigervnc/stable/download_file?file_path=tigervnc-1.8.0.x86_64.tar.gz -O tigervnc-1.8.0.x86_64.tar.gz \ + && tar -zxvf tigervnc-1.8.0.x86_64.tar.gz \ + && mv tigervnc-1.8.0.x86_64 tigervnc + +# Supervisord configuration +COPY files/supervisord_vnc.conf /etc/supervisor/conf.d/ +COPY files/run_vnc.sh /etc/supervisor/conf.d/ +RUN chmod 755 /etc/supervisor/conf.d/run_vnc.sh + + +# Web VNC (noVNC) v0.6.1. +# NOTE: this is a custom version from Doro Wu (fcwu.tw@gmail.com). +# TODO: Check differences and maybe move to 0.6.2 +COPY files/noVNC.tar.gz /usr/lib/ +RUN cd /usr/lib/ && tar -zxvf noVNC.tar.gz +COPY files/index.html /usr/lib/noVNC + +RUN apt-get install -y net-tools + +# Supervisord configuration +COPY files/supervisord_webvnc.conf /etc/supervisor/conf.d/ + +# X environment setup/startup +RUN apt-get install fluxbox -y +COPY files/xstartup /opt/tigervnc/ +RUN chmod 755 /opt/tigervnc/xstartup + +# Prepare for logs +RUN mkdir /metauser/logs && chown metauser:metauser /metauser/logs + +# Rename metauser home folder as a "vanilla" home folder +RUN mv /metauser /metauser_vanilla + +# Add fluxbox customisations +COPY files/dot_fluxbox /metauser_vanilla/.fluxbox +RUN chown -R metauser:metauser /metauser_vanilla/.fluxbox +COPY files/background.jpg /usr/share/images/fluxbox/background.jpg + + +#---------------------- +# Entrypoint +#---------------------- + +# Copy entrypoint +COPY files/entrypoint.sh / + +# Give right permissions +RUN chmod 755 /entrypoint.sh + +# Set entrypoint +ENTRYPOINT ["/entrypoint.sh"] + +# Set user (mainly for Singularity) +USER metauser + +# To access: expose 8590/tcp and 5900/tcp diff --git a/Software/MetaDesktop/build.sh b/Software/MetaDesktop/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..411d42b173cbeb723aea2f27f940e179a747fe02 --- /dev/null +++ b/Software/MetaDesktop/build.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +if [ ! -f "Docker/Dockerfile" ]; then + # TODO: This check is weak: improve me! + echo "Please run this script from the project root folder" + exit 1 +fi + +# Move to the root directory to allow accessing the code in Docker build context +OR_DIR=$PWD +cd Docker + +# Are we on a Git repo? +#echo "" +#git status &> /dev/null +#if [[ "x$?" == "x0" ]] ; then +# CHANGES=$(git status | grep "Changes not staged for commit") +# if [[ "x$CHANGES" == "x" ]] ; then +# TAG=$(git rev-parse HEAD | cut -c1-7) +# echo "I will tag this container with the Git short hash \"$TAG\" " +# else +# TAG="latest" +# echo "You have uncomitted changes, I will not tag this container with the Git short hash. " +# fi +#else +# TAG="latest" +#fi +#echo "" + +# Use --no-cache in case of build problems (i.e. 404 not found) +docker build . -t sarusso/metadesktop + +cd $OR_DIR diff --git a/Software/MetaDesktop/files/background.jpg b/Software/MetaDesktop/files/background.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fe623ec4ddbd6bab4d77dc0f49303d1be040be73 Binary files /dev/null and b/Software/MetaDesktop/files/background.jpg differ diff --git a/Software/MetaDesktop/files/dot_fluxbox/apps b/Software/MetaDesktop/files/dot_fluxbox/apps new file mode 100644 index 0000000000000000000000000000000000000000..7b4aeadb98c6d0d73a3958ab08e83b9b65595e30 --- /dev/null +++ b/Software/MetaDesktop/files/dot_fluxbox/apps @@ -0,0 +1,4 @@ +[app] (name=fbrun) + [Position] (WINCENTER) {0 0} + [Layer] {2} +[end] diff --git a/Software/MetaDesktop/files/dot_fluxbox/init b/Software/MetaDesktop/files/dot_fluxbox/init new file mode 100644 index 0000000000000000000000000000000000000000..6b26e6ea519f1d592db3fdee11aeb7c19dbf8fbe --- /dev/null +++ b/Software/MetaDesktop/files/dot_fluxbox/init @@ -0,0 +1,76 @@ +session.screen0.tabs.usePixmap: true +session.screen0.tabs.maxOver: false +session.screen0.tabs.intitlebar: true +session.screen0.clientMenu.usePixmap: true +session.screen0.iconbar.usePixmap: true +session.screen0.iconbar.iconTextPadding: 10 +session.screen0.iconbar.iconWidth: 128 +session.screen0.iconbar.alignment: Relative +session.screen0.iconbar.mode: {static groups} (workspace) +session.screen0.toolbar.visible: true +session.screen0.toolbar.height: 0 +session.screen0.toolbar.onhead: 1 +session.screen0.toolbar.widthPercent: 100 +session.screen0.toolbar.alpha: 255 +session.screen0.toolbar.maxOver: false +session.screen0.toolbar.autoHide: false +session.screen0.toolbar.layer: Dock +session.screen0.toolbar.placement: BottomCenter +#session.screen0.toolbar.tools: prevworkspace, workspacename, nextworkspace, clock, prevwindow, nextwindow, iconbar, systemtray +session.screen0.toolbar.tools: iconbar, systemtray +session.screen0.menu.alpha: 255 +session.screen0.tab.placement: TopLeft +session.screen0.tab.width: 64 +session.screen0.titlebar.left: Stick +session.screen0.titlebar.right: Minimize Maximize Close +session.screen0.window.focus.alpha: 255 +session.screen0.window.unfocus.alpha: 255 +session.screen0.slit.alpha: 255 +session.screen0.slit.maxOver: false +session.screen0.slit.placement: RightBottom +session.screen0.slit.autoHide: false +session.screen0.slit.acceptKdeDockapps: true +session.screen0.slit.onhead: 0 +session.screen0.slit.layer: Dock +session.screen0.colPlacementDirection: TopToBottom +session.screen0.tabFocusModel: ClickToTabFocus +session.screen0.autoRaise: true +session.screen0.maxDisableMove: false +session.screen0.edgeSnapThreshold: 10 +session.screen0.tooltipDelay: 500 +session.screen0.opaqueMove: true +session.screen0.windowPlacement: RowMinOverlapPlacement +session.screen0.focusNewWindows: true +session.screen0.clickRaises: true +session.screen0.maxDisableResize: false +session.screen0.windowMenu: /metauser/.fluxbox/windowmenu +session.screen0.allowRemoteActions: false +session.screen0.strftimeFormat: %d %b, %a %02k:%M:%S +session.screen0.focusSameHead: false +session.screen0.workspacewarping: true +session.screen0.fullMaximization: false +session.screen0.defaultDeco: NORMAL +session.screen0.noFocusWhileTypingDelay: 0 +session.screen0.menuDelay: 200 +session.screen0.workspaceNames: Workspace 1,Workspace 2,Workspace 3,Workspace 4, +session.screen0.rowPlacementDirection: LeftToRight +session.screen0.focusModel: ClickFocus +session.screen0.showwindowposition: false +session.screen0.maxIgnoreIncrement: true +session.screen0.workspaces: 1 +session.styleOverlay: /metauser/.fluxbox/overlay +session.keyFile: ~/.fluxbox/keys +session.cacheMax: 200 +session.tabsAttachArea: Window +session.slitlistFile: /metauser/.fluxbox/slitlist +session.forcePseudoTransparency: false +session.tabPadding: 0 +session.colorsPerChannel: 4 +session.styleFile: /usr/share/fluxbox/styles//ubuntu-light +session.autoRaiseDelay: 250 +session.cacheLife: 5 +session.appsFile: /metauser/.fluxbox/apps +session.ignoreBorder: false +session.configVersion: 13 +session.doubleClickInterval: 250 +session.menuFile: ~/.fluxbox/menu diff --git a/Software/MetaDesktop/files/dot_fluxbox/keys b/Software/MetaDesktop/files/dot_fluxbox/keys new file mode 100644 index 0000000000000000000000000000000000000000..953d08b263a8422903691069977a3aa56472391c --- /dev/null +++ b/Software/MetaDesktop/files/dot_fluxbox/keys @@ -0,0 +1,137 @@ +# click on the desktop to get menus +OnDesktop Mouse1 :HideMenus +OnDesktop Mouse2 :WorkspaceMenu +OnDesktop Mouse3 :RootMenu + +# scroll on the desktop to change workspaces +OnDesktop Mouse4 :PrevWorkspace +OnDesktop Mouse5 :NextWorkspace + +# scroll on the toolbar to change current window +OnToolbar Mouse4 :PrevWindow {static groups} (iconhidden=no) +OnToolbar Mouse5 :NextWindow {static groups} (iconhidden=no) + +# alt + left/right click to move/resize a window +OnWindow Mod1 Mouse1 :MacroCmd {Raise} {Focus} {StartMoving} +OnWindowBorder Move1 :StartMoving + +OnWindow Mod1 Mouse3 :MacroCmd {Raise} {Focus} {StartResizing NearestCorner} +OnLeftGrip Move1 :StartResizing bottomleft +OnRightGrip Move1 :StartResizing bottomright + +# alt + middle click to lower the window +OnWindow Mod1 Mouse2 :Lower + +# control-click a window's titlebar and drag to attach windows +OnTitlebar Control Mouse1 :StartTabbing + +# double click on the titlebar to shade +OnTitlebar Double Mouse1 :Shade + +# left click on the titlebar to move the window +OnTitlebar Mouse1 :MacroCmd {Raise} {Focus} {ActivateTab} +OnTitlebar Move1 :StartMoving + +# middle click on the titlebar to lower +OnTitlebar Mouse2 :Lower + +# right click on the titlebar for a menu of options +OnTitlebar Mouse3 :WindowMenu + +# alt-tab +Mod1 Tab :NextWindow {groups} (workspace=[current]) +Mod1 Shift Tab :PrevWindow {groups} (workspace=[current]) + +# cycle through tabs in the current window +Mod4 Tab :NextTab +Mod4 Shift Tab :PrevTab + +# go to a specific tab in the current window +Mod4 1 :Tab 1 +Mod4 2 :Tab 2 +Mod4 3 :Tab 3 +Mod4 4 :Tab 4 +Mod4 5 :Tab 5 +Mod4 6 :Tab 6 +Mod4 7 :Tab 7 +Mod4 8 :Tab 8 +Mod4 9 :Tab 9 + +# open a terminal +Mod1 F1 :Exec x-terminal-emulator + +# open a dialog to run programs +Mod1 F2 :Exec fbrun + +# volume settings, using common keycodes +# if these don't work, use xev to find out your real keycodes +176 :Exec amixer sset Master,0 1+ +174 :Exec amixer sset Master,0 1- +160 :Exec amixer sset Master,0 toggle + +# current window commands +Mod1 F4 :Close +Mod1 F5 :Kill +Mod1 F9 :Minimize +Mod1 F10 :Maximize +Mod1 F11 :Fullscreen + +# open the window menu +Mod1 space :WindowMenu + +# exit fluxbox +Control Mod1 Delete :Exit + +# change to previous/next workspace +Control Mod1 Left :PrevWorkspace +Control Mod1 Right :NextWorkspace + +# send the current window to previous/next workspace +Mod4 Left :SendToPrevWorkspace +Mod4 Right :SendToNextWorkspace + +# send the current window and follow it to previous/next workspace +Control Mod4 Left :TakeToPrevWorkspace +Control Mod4 Right :TakeToNextWorkspace + +# change to a specific workspace +Control F1 :Workspace 1 +Control F2 :Workspace 2 +Control F3 :Workspace 3 +Control F4 :Workspace 4 +Control F5 :Workspace 5 +Control F6 :Workspace 6 +Control F7 :Workspace 7 +Control F8 :Workspace 8 +Control F9 :Workspace 9 +Control F10 :Workspace 10 +Control F11 :Workspace 11 +Control F12 :Workspace 12 + +# send the current window to a specific workspace +Mod4 F1 :SendToWorkspace 1 +Mod4 F2 :SendToWorkspace 2 +Mod4 F3 :SendToWorkspace 3 +Mod4 F4 :SendToWorkspace 4 +Mod4 F5 :SendToWorkspace 5 +Mod4 F6 :SendToWorkspace 6 +Mod4 F7 :SendToWorkspace 7 +Mod4 F8 :SendToWorkspace 8 +Mod4 F9 :SendToWorkspace 9 +Mod4 F10 :SendToWorkspace 10 +Mod4 F11 :SendToWorkspace 11 +Mod4 F12 :SendToWorkspace 12 + +# send the current window and change to a specific workspace +Control Mod4 F1 :TakeToWorkspace 1 +Control Mod4 F2 :TakeToWorkspace 2 +Control Mod4 F3 :TakeToWorkspace 3 +Control Mod4 F4 :TakeToWorkspace 4 +Control Mod4 F5 :TakeToWorkspace 5 +Control Mod4 F6 :TakeToWorkspace 6 +Control Mod4 F7 :TakeToWorkspace 7 +Control Mod4 F8 :TakeToWorkspace 8 +Control Mod4 F9 :TakeToWorkspace 9 +Control Mod4 F10 :TakeToWorkspace 10 +Control Mod4 F11 :TakeToWorkspace 11 +Control Mod4 F12 :TakeToWorkspace 12 diff --git a/Software/MetaDesktop/files/dot_fluxbox/lastwallpaper b/Software/MetaDesktop/files/dot_fluxbox/lastwallpaper new file mode 100644 index 0000000000000000000000000000000000000000..1d3a812f04583f48292c28f5c7bec1972f75b17e --- /dev/null +++ b/Software/MetaDesktop/files/dot_fluxbox/lastwallpaper @@ -0,0 +1 @@ +$full $full|/usr/share/images/fluxbox/ubuntu-light.png|style|:0.0 diff --git a/Software/MetaDesktop/files/dot_fluxbox/menu b/Software/MetaDesktop/files/dot_fluxbox/menu new file mode 100644 index 0000000000000000000000000000000000000000..ed1edfbbf100ff08ce6cb48b958a8eb01e0520a6 --- /dev/null +++ b/Software/MetaDesktop/files/dot_fluxbox/menu @@ -0,0 +1,3 @@ +[begin] (fluxbox) +[include] (/etc/X11/fluxbox/fluxbox-menu) +[end] diff --git a/Software/MetaDesktop/files/dot_fluxbox/overlay b/Software/MetaDesktop/files/dot_fluxbox/overlay new file mode 100644 index 0000000000000000000000000000000000000000..4ddc46b04c8c8d4b357a950dc0ff246d7960cbad --- /dev/null +++ b/Software/MetaDesktop/files/dot_fluxbox/overlay @@ -0,0 +1,4 @@ +! The following line will prevent styles from setting the background. +! background: none +background: aspect +background.pixmap: /usr/share/images/fluxbox/background.jpg diff --git a/Software/MetaDesktop/files/dot_fluxbox/windowmenu b/Software/MetaDesktop/files/dot_fluxbox/windowmenu new file mode 100644 index 0000000000000000000000000000000000000000..d867b64c2299e4d714323ec3018ce3efe3a4f466 --- /dev/null +++ b/Software/MetaDesktop/files/dot_fluxbox/windowmenu @@ -0,0 +1,15 @@ +[begin] + [shade] + [stick] + [maximize] + [iconify] + [raise] + [lower] + [settitledialog] + [sendto] + [layer] + [alpha] + [extramenus] + [separator] + [close] +[end] diff --git a/Software/MetaDesktop/files/entrypoint.sh b/Software/MetaDesktop/files/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..efe8414b605e6a4c743c2b0164cadf67c723ce05 --- /dev/null +++ b/Software/MetaDesktop/files/entrypoint.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Exit on any error. More complex thing could be done in future +# (see https://stackoverflow.com/questions/4381618/exit-a-script-on-error) +set -e + +echo "" +echo "[INFO] Executing entrypoint..." + +#--------------------- +# Setup home +#--------------------- + +# First try without sudo (Singularity with --writable-tmpfs), then sudo (Docker) +echo "[INFO] Setting up home" + +# Get immune to -e inside the curly brackets +{ + cp -a /metauser_vanilla /metauser &> /dev/null + EXIT_CODE=$? +} || true + +# Check if the above failed and we thus have to use sudo +if [ "$EXIT_CODE" != "0" ]; then + #echo "Using sudo" + sudo cp -a /metauser_vanilla /metauser +fi + +# Manually set home (mainly for Singularity) +echo "[INFO] Setting up HOME env var" +export HOME=/metauser +cd /metauser + +#--------------------- +# Save env +#--------------------- +echo "[INFO] Dumping env" + +# Save env vars for later usage (e.g. ssh) + +env | \ +while read env_var; do + if [[ $env_var == HOME\=* ]]; then + : # Skip HOME var + elif [[ $env_var == PWD\=* ]]; then + : # Skip PWD var + else + echo "export $env_var" >> /tmp/env.sh + fi +done + +#--------------------- +# Entrypoint command +#--------------------- + +if [ "$@x" == "x" ]; then + DEFAULT_COMMAND="supervisord -c /etc/supervisor/supervisord.conf" + echo -n "[INFO] Executing default entrypoint command: " + echo $DEFAULT_COMMAND + exec $DEFAULT_COMMAND +else + echo -n "[INFO] Executing entrypoint command: " + echo $@ + exec $@ +fi + + diff --git a/Software/MetaDesktop/files/index.html b/Software/MetaDesktop/files/index.html new file mode 100644 index 0000000000000000000000000000000000000000..6a6c57d819e940020afaaef0023fa1a788867147 --- /dev/null +++ b/Software/MetaDesktop/files/index.html @@ -0,0 +1,12 @@ +<html> +<head> +<script type="text/javascript"> +function redirecter(){ + window.location = "./vnc.html?autoconnect=true&resize=remote" +} +</script> +</head> +<body onLoad="redirecter()"> +Access VNC: click <a href="./vnc.html?autoconnect=true&resize=remote">here</a>. +</body> +</html> \ No newline at end of file diff --git a/Software/MetaDesktop/files/noVNC.tar.gz b/Software/MetaDesktop/files/noVNC.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..a531bed617c54af4826c7afd76192bc2736100ba Binary files /dev/null and b/Software/MetaDesktop/files/noVNC.tar.gz differ diff --git a/Software/MetaDesktop/files/run_vnc.sh b/Software/MetaDesktop/files/run_vnc.sh new file mode 100755 index 0000000000000000000000000000000000000000..628f8e88542e75a9918b98db99d8cfdcd6cac281 --- /dev/null +++ b/Software/MetaDesktop/files/run_vnc.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Exec TigerVNC server +/opt/tigervnc/usr/bin/vncserver :0 -SecurityTypes None -xstartup /opt/tigervnc/xstartup + +# Check it is running. If it is not, exit +while true +do + + PSOUT=$(ps -ef | grep /opt/tigervnc/usr/bin/Xvnc | grep SecurityTypes) + + if [[ "x$PSOUT" == "x" ]] ; then + exit 1 + fi + + # Sleep other 10 secs before re-checking + sleep 10 + +done diff --git a/Software/MetaDesktop/files/sudoers b/Software/MetaDesktop/files/sudoers new file mode 100644 index 0000000000000000000000000000000000000000..47ab37c90fdec1df833409f825d2665fe7d1f899 --- /dev/null +++ b/Software/MetaDesktop/files/sudoers @@ -0,0 +1,30 @@ +# +# This file MUST be edited with the 'visudo' command as root. +# +# Please consider adding local content in /etc/sudoers.d/ instead of +# directly modifying this file. +# +# See the man page for details on how to write a sudoers file. +# +Defaults env_reset +Defaults mail_badpass +Defaults secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +# Host alias specification + +# User alias specification + +# Cmnd alias specification + +# User privilege specification +root ALL=(ALL:ALL) ALL + +# Members of the admin group may gain root privileges +%admin ALL=(ALL) ALL + +# Allow members of group sudo to execute any command +%sudo ALL=(ALL:ALL) NOPASSWD:ALL + +# See sudoers(5) for more information on "#include" directives: + +#includedir /etc/sudoers.d diff --git a/Software/MetaDesktop/files/supervisord.conf b/Software/MetaDesktop/files/supervisord.conf new file mode 100644 index 0000000000000000000000000000000000000000..28b8cc96a8f447b99391250055d46dab5f6e3b56 --- /dev/null +++ b/Software/MetaDesktop/files/supervisord.conf @@ -0,0 +1,29 @@ +; supervisor config file (modified for our own purpose) + +[unix_http_server] +file=/metauser/.supervisor.sock ; (the path to the socket file) +chmod=0700 ; sockef file mode (default 0700) + +[supervisord] +logfile=/metauser/logs/supervisord.log ; (main log file;default $CWD/supervisord.log) +pidfile=/metauser/logs/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +childlogdir=/metauser/logs ; ('AUTO' child log dir, default $TEMP) +nodaemon=true ; Mandatory to run Supervisor in foreground and avoid Docker to exit! + +; The below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///metauser/.supervisor.sock ; use a unix:// URL for a unix socket + +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. + +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/Software/MetaDesktop/files/supervisord_vnc.conf b/Software/MetaDesktop/files/supervisord_vnc.conf new file mode 100644 index 0000000000000000000000000000000000000000..782c7ffa08dcdb2a8bed029a967603ff8445b923 --- /dev/null +++ b/Software/MetaDesktop/files/supervisord_vnc.conf @@ -0,0 +1,23 @@ +;======================================= +; VNC service +;======================================= + +[program:vnc] + +; General +directory = / +command = /etc/supervisor/conf.d/run_vnc.sh +numprocs = 1 +autostart = true +autorestart = true +startsecs = 10 +stopwaitsecs = 30 +process_name = vnc + +; Standard out / error +stdout_logfile = /metauser/logs/%(program_name)s.log +stdout_logfile_maxbytes = 5MB +stdout_logfile_backups = 10 +stderr_logfile = /metauser/logs/%(program_name)s.log +stderr_logfile_maxbytes = 5MB +stderr_logfile_backups = 10 diff --git a/Software/MetaDesktop/files/supervisord_webvnc.conf b/Software/MetaDesktop/files/supervisord_webvnc.conf new file mode 100644 index 0000000000000000000000000000000000000000..302d9d918521494638c7e8bd0761476b4fbe9a4c --- /dev/null +++ b/Software/MetaDesktop/files/supervisord_webvnc.conf @@ -0,0 +1,23 @@ +;======================================= +; noVNC service +;======================================= + +[program:novnc] + +; General +directory = /usr/lib/noVNC/ +command = /usr/lib/noVNC/utils/launch.sh --listen 8590 +numprocs = 1 +autostart = true +autorestart = true +startsecs = 10 +stopwaitsecs = 30 +process_name = novnc + +; Standard out / error +stdout_logfile = /metauser/logs/%(program_name)s.log +stdout_logfile_maxbytes = 5MB +stdout_logfile_backups = 10 +stderr_logfile = /metauser/logs/%(program_name)s.log +stderr_logfile_maxbytes = 5MB +stderr_logfile_backups = 10 diff --git a/Software/MetaDesktop/files/xstartup b/Software/MetaDesktop/files/xstartup new file mode 100644 index 0000000000000000000000000000000000000000..26ed61798eb711928c016f51c20e23fa1cf9605c --- /dev/null +++ b/Software/MetaDesktop/files/xstartup @@ -0,0 +1,29 @@ +#!/bin/sh + +cd /metauser + +unset SESSION_MANAGER +unset DBUS_SESSION_BUS_ADDRESS +OS=`uname -s` +if [ $OS = 'Linux' ]; then + case "$WINDOWMANAGER" in + *gnome*) + if [ -e /etc/SuSE-release ]; then + PATH=$PATH:/opt/gnome/bin + export PATH + fi + ;; + esac +fi +if [ -x /etc/X11/xinit/xinitrc ]; then + exec /etc/X11/xinit/xinitrc +fi +if [ -f /etc/X11/xinit/xinitrc ]; then + exec sh /etc/X11/xinit/xinitrc +fi +[ -r $HOME/.Xresources ] && xrdb $HOME/.Xresources +#xsetroot -solid grey +xterm -geometry 80x24+10+10 -ls -title "$VNCDESKTOP Desktop" & + +#twm & +fluxbox & diff --git a/Software/MetaDesktop/run.sh b/Software/MetaDesktop/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..165a7752531a8785917c3ecab34384a5c9e4ec29 --- /dev/null +++ b/Software/MetaDesktop/run.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker run -v$PWD/:/data -p5900:5900 -p8590:8590 -it sarusso/metadesktop diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..474fb751ac908b6fc612bb03928b0651e47233ac --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +version: '3' +services: + + slurmclustermaster-main: + image: "rosetta/slurmclustermaster" + container_name: slurmclustermaster-main + hostname: slurmclustermaster-main + environment: + - SAFEMODE=False + privileged: true + volumes: + - ./data/shared:/shared + # - ./data/singularity_cache:/rosetta/.singularity/cache # Not working, check permissions... + ports: + - "8590:8590" + - "5900:5900" + + slurmclusterworker-one: + image: "rosetta/slurmclusterworker" + container_name: slurmclusterworker-one + hostname: slurmclusterworker-one + environment: + - SAFEMODE=False + privileged: true + volumes: + - ./data/shared:/shared + + dregistry: + container_name: dregistry + hostname: dregistry + image: "rosetta/dregistry" + volumes: + - ./data/dregistry:/var/lib/registry + ports: + - "5000:5000" diff --git a/examples/build_tag_push_run-metadesktop.sh b/examples/build_tag_push_run-metadesktop.sh new file mode 100755 index 0000000000000000000000000000000000000000..55c87811ae34ccc6d88f816f548753e3bbdea25a --- /dev/null +++ b/examples/build_tag_push_run-metadesktop.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + + +# Build +cd ../Software/MetaDesktop +docker build . -t rosetta/metadesktop +cd ../../ + +# Tag +docker tag rosetta/metadesktop localhost:5000/rosetta/metadesktop + +# Push +docker push localhost:5000/rosetta/metadesktop + +# Run +rosetta/shell slurmclustermaster-main "SINGULARITY_NOHTTPS=true singularity run --writable-tmpfs --containall --cleanenv docker://dregistry:5000/rosetta/metadesktop" + +# Run variants/tests +# rosetta/shell slurmclustermaster-main "SINGULARITY_NOHTTPS=true singularity run docker://dregistry:5000/rosetta/metadesktop" +# rosetta/shell slurmclustermaster-main "rm -rf tmp && mkdir tmp && SINGULARITYENV_HOME=/metauser SINGULARITY_NOHTTPS=true singularity run -B ./tmp:/tmp,./tmp:/metauser --writable-tmpfs --containall --cleanenv docker://dregistry:5000/rosetta/metadesktop" + +cd examples diff --git a/images/base/Dockerfile b/images/base/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..8ad1438844046be0220f4eefd0f7a286b9874033 --- /dev/null +++ b/images/base/Dockerfile @@ -0,0 +1,146 @@ +FROM ubuntu:18.04 +MAINTAINER Stefano Alberto Russo <stefano.russo@gmail.com> + +#---------------------- +# Basics +#---------------------- + +# Set non-interactive +ENV DEBIAN_FRONTEND noninteractive + +# Update +RUN apt-get update + +# Utilities +RUN apt-get install -y nano telnet unzip wget supervisor openssh-server + +# Devel +RUN apt-get install -y build-essential python-dev git-core + +# Java +RUN apt-get install -y openjdk-8-jre + +# IP utilities (mandatory for DNS!) +RUN apt-get install net-tools iproute2 iputils-ping -y + + +#------------------------ +# Scienceuser user +#------------------------ + +# Add group. We chose GID 65527 to try avoiding conflicts. +RUN groupadd -g 65527 rosetta + +# Add user. We chose UID 65527 to try avoiding conflicts. +RUN useradd rosetta -d /rosetta -u 65527 -g 65527 -m -s /bin/bash + +# Add rosetta user to sudoers +RUN adduser rosetta sudo + +# Keys +RUN mkdir /rosetta/.ssh +COPY keys/authorized_keys /rosetta/.ssh/ +COPY keys/id_rsa /rosetta/.ssh/ +RUN chmod 0600 /rosetta/.ssh/id_rsa +COPY keys/id_rsa.pub /rosetta/.ssh/ +RUN chown -R rosetta:rosetta /rosetta/.ssh + +# Install suodo +RUN apt-get install sudo -y + +# No pass sudo (for everyone, actually) +COPY sudoers /etc/sudoers + +# bash_profile for loading correct env (/env.sh created by entrypoint.sh) +RUN echo "source /env.sh" > /rosetta/.bash_profile + + +#------------------------ +# Data, Logs and opt dirs +#------------------------ + +# Create dirs +RUN mkdir /data && mkdir /opt/rosetta && mkdir /var/log/rosetta + +# Give right permissions +RUN chown -R rosetta:rosetta /data && chown -R rosetta:rosetta /opt/rosetta && chown -R rosetta:rosetta /var/log/rosetta + + +#---------------------- +# Supervisord conf +#---------------------- + +COPY supervisord.conf /etc/supervisor/ + + +#---------------------- +# SSH conf +#---------------------- + +RUN mkdir /var/run/sshd && chmod 0755 /var/run/sshd +COPY supervisord_sshd.conf /etc/supervisor/conf.d/ + + +#---------------------- +# Prestartup scripts +#---------------------- + +# Create dir for prestartup scripts and copy main script +RUN mkdir /prestartup +COPY prestartup.py / + + +#---------------------- +# Singularity +#---------------------- + +# Dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config \ + cryptsetup-bin \ + wget + +# Install GO +RUN cd /tmp && wget https://dl.google.com/go/go1.11.linux-amd64.tar.gz +RUN cd /tmp && tar -zxf go1.11.linux-amd64.tar.gz && mv go /usr/local +ENV GOROOT=/usr/local/go +ENV GOPATH=/root/go +ENV PATH=$PATH:/usr/local/go/bin:$GOPATH/bin + +# Install Singularity +RUN mkdir -p /usr/local/var/singularity/mnt && \ + mkdir -p $GOPATH/src/github.com/sylabs && \ + cd $GOPATH/src/github.com/sylabs && \ + wget https://github.com/sylabs/singularity/releases/download/v3.4.1/singularity-3.4.1.tar.gz && \ + tar -xzvf singularity-3.4.1.tar.gz +RUN cd $GOPATH/src/github.com/sylabs/singularity && \ + ./mconfig -p /usr/local && \ + make -C builddir && \ + make -C builddir install + +# Build test image +RUN mkdir /singularity_images && chmod 777 /singularity_images +COPY testimage.def /singularity_images/testimage.def +RUN singularity build /singularity_images/testimage.simg /singularity_images/testimage.def + + +#---------------------- +# Entrypoint +#---------------------- + +# Copy entrypoint +COPY entrypoint.sh / + +# Give right permissions +RUN chmod 755 /entrypoint.sh + +# Set entrypoint +ENTRYPOINT ["/entrypoint.sh"] + + diff --git a/images/base/entrypoint.sh b/images/base/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..b3c80a519c188b7b29d85e8cc1069c7781a78e32 --- /dev/null +++ b/images/base/entrypoint.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# Exit on any error. More complex thing could be done in future +# (see https://stackoverflow.com/questions/4381618/exit-a-script-on-error) +set -e + +echo "" +echo "[INFO] Executing entrypoint..." + +#--------------------- +# Persistency +#--------------------- + +echo "[INFO] Handling safe persistency" +if [ "x$SAFE_PERSISTENCY" == "xTrue" ]; then + echo "[INFO] Safe persistency set" + if [ ! -f /safe_persistent/persistent.img ]; then + truncate -s 10G /safe_persistent/persistent.img + mkfs.ext4 -F /safe_persistent/persistent.img + fi + mkdir /persistent + mount -oloop /safe_persistent/persistent.img /persistent +fi + + +echo "[INFO] Handling persistency" + +# If persistent data: +if [ "x$PERSISTENT_DATA" == "xTrue" ]; then + echo "[INFO] Persistent data set" + if [ ! -f /persistent/data/.persistent_initialized ]; then + mv /data /persistent/data + ln -s /persistent/data /data + touch /data/.persistent_initialized + else + mkdir -p /trash + mv /data /trash + ln -s /persistent/data /data + fi +fi + +# If persistent log: +if [ "x$PERSISTENT_LOG" == "xTrue" ]; then + echo "[INFO] Persistent log set" + if [ ! -f /persistent/log/.persistent_initialized ]; then + mv /var/log /persistent/log + ln -s /persistent/log /var/log + touch /var/log/.persistent_initialized + else + mkdir -p /trash + mv /var/log /trash + ln -s /persistent/log /var/log + fi +fi + +# If persistent home: +if [ "x$PERSISTENT_HOME" == "xTrue" ]; then + echo "[INFO] Persistent home set" + if [ ! -f /persistent/home/.persistent_initialized ]; then + mv /home /persistent/home + ln -s /persistent/home /home + touch /home/.persistent_initialized + else + mkdir -p /trash + mv /home /trash + ln -s /persistent/home /home + fi +fi + + +# If persistent opt: +if [ "x$PERSISTENT_OPT" == "xTrue" ]; then + echo "[INFO] Persistent opt set" + if [ ! -f /persistent/opt/.persistent_initialized ]; then + mv /opt /persistent/opt + ln -s /persistent/opt /opt + touch /opt/.persistent_initialized + else + mkdir -p /trash + mv /opt /trash + ln -s /persistent/opt /opt + fi +fi + + +#--------------------- +# Prestartup scripts +#--------------------- + +if [ "x$SAFEMODE" == "xFalse" ]; then + echo "[INFO] Executing prestartup scripts (current + parents):" + python /prestartup.py +else + echo "[INFO] Not executing prestartup scripts as we are in safemode" +fi + + +#--------------------- +# Save env +#--------------------- +echo "[INFO] Dumping env" + +# Save env vars for later usage (e.g. ssh) + +env | \ +while read env_var; do + if [[ $env_var == HOME\=* ]]; then + : # Skip HOME var + elif [[ $env_var == PWD\=* ]]; then + : # Skip PWD var + else + echo "export $env_var" >> /env.sh + fi +done + +#--------------------- +# Entrypoint command +#--------------------- +# Start! + + +if [[ "x$@" == "x" ]] ; then + ENTRYPOINT_COMMAND="supervisord" +else + ENTRYPOINT_COMMAND=$@ +fi + +echo -n "[INFO] Executing Docker entrypoint command: " +echo $ENTRYPOINT_COMMAND +exec "$ENTRYPOINT_COMMAND" diff --git a/images/base/keys/authorized_keys b/images/base/keys/authorized_keys new file mode 100644 index 0000000000000000000000000000000000000000..e4e1df61a89a45e50620f4bc74ba5547481ecd2d --- /dev/null +++ b/images/base/keys/authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== docker@dev.ops diff --git a/images/base/keys/id_rsa b/images/base/keys/id_rsa new file mode 100644 index 0000000000000000000000000000000000000000..de5d6bf2a5ab7f74a4f706c7be5b0346b14ea73c --- /dev/null +++ b/images/base/keys/id_rsa @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEAtp+MIi4kZhNbJWufsNCFt8MD1v5qoYZJuyMggUvqxk4J7bO8 +bFnMaG72pjXQ+ikizZ4xWWwW+2Gn/wb1q8EIHaiBVVN4EVh2M6FOo7PpSrQY3dHt +gwFRor1gihJEhi8e84ce7h/ybfJMmeDWmatKefkhRwc3j3Uf9a1EXfBvPhtzmgjy +qj/iUE+ETFEBwuUjUSS3CtaYmHZoQ4mBMrkBuBZUrSgliwl3lGNhFYqzTHkCHzoG +1BybFYx9O85tcvMu5U1gzFlWTp5BugKeZAfp+JdXlm7JTioA79vSUxTjFPm7xr2G +Ye9JZiyux4OAjKV1YL8OVRfb/e2IoooC8Lwu8GrAR2gEsB+8FKmpF2IXRhcSdt8V +TEi57yZ+EvvuC/LfG4n0SFaoU+rWQjzlOH3TrySgkYyq3+4HFvxN96L2RYKDgidr +wSyIfTfTBNL+7RteSaUbYCAbAl8C3Jd7VcgmSf3smDG3N0CRrG3Dxm9gmEYw23rz +wkJTWr3DHZcqzSJHu1YpjOWlvDgSSlYecAjUaXfV8payqNXGR2Mbv3OS6+lVGYy6 +cQ5Us/iLk5+FRO75oBzT513ZcUSoyxxvxWa2U0glhRAXEaEmv4crC0hcMA4ynfZ3 +pTmUTQbyfjWOPLwPIsVZ5OFgDJUf+aoY00WpuugVcrB3kXG4ydiBo8EyQuUCAwEA +AQKCAgEAh0Vm52qGS5XKzc0KXE4YviUVkwqgsURnGNbMHPm+zWTAtfGMgDWD01de +G3+Ba8tMnEGxDCukWk/bwGvHTZGOEWnfYvSQ20hLRbMWLOv2wf7k7GmzJHa1oXXl +LGCboUkGBBzyLDA9wnLXiqOgUfMvF2oR3CrcXMbFBZVyLqMJw1dSKaa3GKR5XkOI +G39lbpeLsW8gpkaOgWAzmtMfgBLJ0zG3RwuVw4cfrCpwnyQ960c26ypwJG2L8ko9 ++S7Oo3a+JdtK+BK0e0d+J+oIqM+z3w87MZKeSeeTChgpkqDGE6NoE64O/DvigmxW +ijI95fApIaBjXWRu74gizUKtKuQ5X1pvo1zyQXWqhcaFnB4fv7+kI4L7JwlY4QIf +CLEjYfZFXCtmRo6QPn/09OPiU8xgimqVdIfr7JYjDMoEyMW9vfy5EJmtwS9M41tJ +2gDbhw1fhwUVW1MsJjLuboMXudsubGvGUy+jB48YPQs2Yx13NgUu15jtvPxVCC9v +CdnaL6PJtloSXh5zYpapUg2UN5oH48BLw1hWFoDBcgzTxlCjyEJGtem9QM1Y997e +z561gw8iu1vw0XDuv5zd7qzyIgAYuB8b3Pe6Rg+V2jennKvymMrtCvUNcLRs1pF8 +LV0t9rTQzQWP5d8AmxywZfgXaQ0zcrTTd2rkjwf/yBH5yNIhDAECggEBAOl6K4pA +EHsWjGy1IrvhoDztbLkkLzxLvVbkjKT9NJwHucL+oekfxLt/c+1+8mNUjiMHyLd8 +cH+R2Lyy1YhfBrT92gPfRRHUBLx+XS0P3p0dj3N+U+C//WAaMS5mb+pkTUFGLQ8g +vRHPHt0rAjvzpMUCNUtO+o11srZIOjLOLYkxSIDqwFXFWDyCgfqYev1jkNDivILk +HjeNrz3G5XpIBQdclZtX1f9yII5EfA6ChUGOLIAMwY1Mr6gTJTKtE3Q6anC0AgoW +ugw5oTSZpKySCKjf20AVcKvPBA3Tq+TBR10XmSTwL6r0bzuptXJBr+teOsnvs1+g +qhwgqExgFrkLf30CggEBAMg9g5VtYtmSfKxFR/59YSwjgRUSz7xFzsdUnbYN71X1 +fd7o5htmEIitGRshzbRLCE85cW6TlGC02RiH9wQctc288BkYPYMW7LXRluKSXm+J +WlEwiWsGAJzsNK8U6fxCM0ZsX2AQ3tVSRHnhl7a/CByUQZFS/M+aTLWuOZQZElyK +PqsCw4eD6GbTk2qtlkxp8Gc/eAnii4KWfb6yvx5dgJXz1Nuu/ueZI+lmEP+EgubD +/f9vUzNFHgcU0+z2bH49gvUJ6t9nIAJ4HsHvoI6L286YVzR7qCP5inVksRspVLPP +iH8EDr4QhLnCh4GZiWy1JBpm/Zg+YcibQKxacs/nfYkCggEAXby3DmJ6O3DqIBr5 +PwVvGAcax5pHfKXL9r772KHwJVTUt/0TdE1U5xJcsNVu64JfLqFJbKGBaTZdFiWW +pZHBV5kzlqplSKse267AKf9dGSdtGKl3c5yhVZwucrqd5DUw7ywFmzVBs4y8j39c +/kTruk0QqJOk9HZ0scp90zgEADjRKzEU11rL+j9LgBkICAOZeMQPe12q5BL2cI8S +Qu33VuVNC3lQaaage33zcL/mUFOMejyk2N4ZCBnnrVjfnqJ1aZpb10EYoR/iIQQu +oTpgT6zQkgIJonES55o8QTN4O1/mFHZ6LODGZ+XS+3Rz9MN4Rur90T7oDTLvXvqV +JOYA4QKCAQEAluueKFq4nUnGQ8U3/Pyc57qeyLZT8hAfSKdi8ttP31bXFtIs1Mu5 +fHoSqRtyQggnbCbccr4yoCzOT6nyqJvG/xj/UbquagY2RNeCRKSTHrfEZdsSR6LP +hXaWQrudm659nP+DZxFwEhIeYEqCoY8b2wZ24MROnV4roOd+qDu5VhwwHY5ItvPZ +jt66hjXtSQyzz+3LWI/yHGu2vKtWVtmcV+jeLvGXWBFZOsnd1+gVDT79Sq+qYsMe +XbH6BOi6Xu+Xq35dEyJTwuisLfmg5q9M7Uput7TXxr2G+PH6doFRQPETbMAvKFuk +3albnneNV2yzmF61ljC2XI9/UCgfzskoGQKCAQBcgsPCQREaEiMvfmWjoDeip/Cy +c0QzTJ6Oy5kVxfjHxRhEZyjKPBbXLGjewLoUfuBJvOJ7Iqadv5vP2AOUS0KMkmwt +w0rIUhk9WaLo+f4Fci1d14CPs59w2GYhSniGOT/qiPprUZVUr+J0fJ6q2i7kRUTR +gLmSxLEKbHUTKJVTJ0wviIHZYHA+WIQzK1j2NdVIjpLNRXaV4+g0vDBnmCovbBgy +VkyXcPF8q/aDjPcDb9cyCxt4PJQRrP7n959Y2sIjyVwAIEg5wzFuPp3LG+ITnLpG +TtrkLRzqxPKqAY0p4D/7exFyk4SeUHFWWifs7uYeflw3vxN+VmazFE4WdXh3 +-----END RSA PRIVATE KEY----- diff --git a/images/base/keys/id_rsa.pub b/images/base/keys/id_rsa.pub new file mode 100644 index 0000000000000000000000000000000000000000..e4e1df61a89a45e50620f4bc74ba5547481ecd2d --- /dev/null +++ b/images/base/keys/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== docker@dev.ops diff --git a/images/base/prestartup.py b/images/base/prestartup.py new file mode 100644 index 0000000000000000000000000000000000000000..82eb0ec871677b25ab0cf65c9d9fc87d14382fa3 --- /dev/null +++ b/images/base/prestartup.py @@ -0,0 +1,89 @@ + +import os +import sys +import datetime +import subprocess +from collections import namedtuple + +def shell(command, interactive=False): + '''Execute a command in the shell. By default prints everything. If the capture switch is set, + then it returns a namedtuple with stdout, stderr, and exit code.''' + + if interactive: + exit_code = subprocess.call(command, shell=True) + if exit_code == 0: + return True + else: + return False + + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (stdout, stderr) = process.communicate() + exit_code = process.wait() + + # Convert to str (Python 3) + stdout = stdout.decode(encoding='UTF-8') + stderr = stderr.decode(encoding='UTF-8') + + # Output namedtuple + Output = namedtuple('Output', 'stdout stderr exit_code') + + # Return + return Output(stdout, stderr, exit_code) + + +prestartup_scripts_path='/prestartup' +def sorted_ls(path): + mtime = lambda f: os.stat(os.path.join(path, f)).st_mtime + return list(sorted(os.listdir(path), key=mtime)) + +for item in sorted_ls(prestartup_scripts_path): + if item.endswith('.sh'): + + # Execute this startup script + print('[INFO] Executing prestartup script "{}"...'.format(item)) + script = prestartup_scripts_path+'/'+item + + # Use bash and not chmod + execute, see https://github.com/moby/moby/issues/9547 + out = shell('bash {}'.format(script)) + + # Set date + date_str = str(datetime.datetime.now()).split('.')[0] + + # Print and log stdout and stderr + for line in out.stdout.strip().split('\n'): + print(' out: {}'.format(line)) + + for line in out.stderr.strip().split('\n'): + print(' err: {}'.format(line)) + + # Handle error in the startup script + if out.exit_code: + print('[ERROR] Exit code "{}" for "{}"'.format(out.exit_code, item)) + + # Exit with error code 1 + sys.exit(1) + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/images/base/sudoers b/images/base/sudoers new file mode 100644 index 0000000000000000000000000000000000000000..47ab37c90fdec1df833409f825d2665fe7d1f899 --- /dev/null +++ b/images/base/sudoers @@ -0,0 +1,30 @@ +# +# This file MUST be edited with the 'visudo' command as root. +# +# Please consider adding local content in /etc/sudoers.d/ instead of +# directly modifying this file. +# +# See the man page for details on how to write a sudoers file. +# +Defaults env_reset +Defaults mail_badpass +Defaults secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +# Host alias specification + +# User alias specification + +# Cmnd alias specification + +# User privilege specification +root ALL=(ALL:ALL) ALL + +# Members of the admin group may gain root privileges +%admin ALL=(ALL) ALL + +# Allow members of group sudo to execute any command +%sudo ALL=(ALL:ALL) NOPASSWD:ALL + +# See sudoers(5) for more information on "#include" directives: + +#includedir /etc/sudoers.d diff --git a/images/base/supervisord.conf b/images/base/supervisord.conf new file mode 100644 index 0000000000000000000000000000000000000000..2f4678dd2cbe17a94e6489d14d47e0f0212383ce --- /dev/null +++ b/images/base/supervisord.conf @@ -0,0 +1,34 @@ +; supervisor config file + +[unix_http_server] +file=/var/run/supervisor.sock ; (the path to the socket file) +chmod=0700 ; sockef file mode (default 0700) + +[supervisord] +logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) +pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) +nodaemon=true ; Mandatory to run Supervisor in foreground and avoid Docker to exit! + +; The below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL for a unix socket + +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. + +[include] +files = /etc/supervisor/conf.d/*.conf + + + + + diff --git a/images/base/supervisord_sshd.conf b/images/base/supervisord_sshd.conf new file mode 100644 index 0000000000000000000000000000000000000000..b192539ea4cc0f90dc21d022f2304bedad757a54 --- /dev/null +++ b/images/base/supervisord_sshd.conf @@ -0,0 +1,17 @@ +[program:sshd] + +; Process definition +process_name = sshd +command = /usr/sbin/sshd -D +autostart = true +autorestart = true +startsecs = 5 +stopwaitsecs = 10 + +; Log files +stdout_logfile = /var/log/supervisor/%(program_name)s_out.log +stdout_logfile_maxbytes = 10MB +stdout_logfile_backups = 5 +stderr_logfile = /var/log/supervisor/%(program_name)s_err.log +stderr_logfile_maxbytes = 10MB +stderr_logfile_backups = 5 diff --git a/images/base/testimage.def b/images/base/testimage.def new file mode 100644 index 0000000000000000000000000000000000000000..adb46a6fdb6cbb7d464a8ac526e610e3f9720f74 --- /dev/null +++ b/images/base/testimage.def @@ -0,0 +1,12 @@ +Bootstrap:library +From: debian:9 + +%runscript + exec echo "Hello world!" + +%environment + TESTVARIABLE=42 + export TESTVARIABLE + +%labels + AUTHOR stefano.russo@gmail.com diff --git a/images/default.conf b/images/default.conf new file mode 100755 index 0000000000000000000000000000000000000000..0b5070dd453c82c50de9a2798ea6e5c85add89ec --- /dev/null +++ b/images/default.conf @@ -0,0 +1,49 @@ +[ + { + "service": "dns", + "instance": "exposed", + "sleep": 5, + "hostname": "dns" + }, + { + "service": "slurmclusterworker", + "instance": "partition1node1", + "instance_type": "standard", + "sleep": 0, + "links": [ + { + "name": "DNS", + "service": "dns", + "instance": null + } + ], + "hostname": "partition1-node1" + }, + { + "service": "slurmclusterworker", + "instance": "partition2node1", + "instance_type": "standard", + "sleep": 0, + "links": [ + { + "name": "DNS", + "service": "dns", + "instance": null + } + ], + "hostname": "partition2-node1" + }, + { + "service": "slurmclustermaster", + "instance": "one", + "sleep": 0, + "links": [ + { + "name": "DNS", + "service": "dns", + "instance": null + } + ], + "hostname": "slurm-master" + } +] diff --git a/images/dregistry/Dockerfile b/images/dregistry/Dockerfile new file mode 100755 index 0000000000000000000000000000000000000000..424e5383345954885c38eee4d9326de1d7045de3 --- /dev/null +++ b/images/dregistry/Dockerfile @@ -0,0 +1 @@ +FROM registry:2 diff --git a/images/slurmbase/Dockerfile b/images/slurmbase/Dockerfile new file mode 100755 index 0000000000000000000000000000000000000000..8297d228cf4fbf96947ba29171bd7c243a75c834 --- /dev/null +++ b/images/slurmbase/Dockerfile @@ -0,0 +1,24 @@ +FROM rosetta/base +MAINTAINER Stefano Alberto Russo <stefano.russo@gmail.com> + +# Install Slurm +RUN apt-get -y install slurm-wlm + +# Explicitly create /var/run/ dirs +RUN mkdir -p /var/run/munge +RUN mkdir -p /var/run/slurm-wlm + +# Add munge key and set permissions +COPY munge.key /etc/munge/munge.key +RUN chown munge:munge /etc/munge/munge.key +RUN chmod 0400 /etc/munge/munge.key + +# Add munge daemon supervisord coinf +COPY supervisord_munge.conf /etc/supervisor/conf.d/ + +# Add Slurm conf +COPY slurm.conf /etc/slurm-llnl/slurm.conf + +# TODO: why do we need this? +RUN ln -s /var/lib/slurm-llnl /var/lib/slurm-wlm +RUN ln -s /var/log/slurm-llnl /var/log/slurm-wlm diff --git a/images/slurmbase/munge.key b/images/slurmbase/munge.key new file mode 100755 index 0000000000000000000000000000000000000000..6eacfca5ef28972de46794927a3e0bfabfa18217 --- /dev/null +++ b/images/slurmbase/munge.key @@ -0,0 +1,18 @@ +nVSHnJzgIVuvfxPf2AJtHlEA6jWeJdLhxdujd04/ZPysw3MZk7QaURlIOMArbXpH8w37bsuAFw9G +ulS6bWPIaja8JiLYqjdQQApLBhCaqK27nid3siIiZkzU2J4IEaCV3KneOPZax9xuIJyDIHb5ailI +V7YQXddTprZt/uluhkAYNwaVQt6PvXLH3Kofa2M/rEPVMc8VaYgsmHq3GGkwqZ3tqNyE0GienWal +zM56vJJrUZtEPc/IK8Sl/0QRkCWXxLOr1XRrNf6w9CbI3Wx44LF22L0NcaV6WLYDHbIn2raTod+5 +nSHD0Mrnvmvx70kcKCsDOftcoj4d8eRlRQqnY0hM2NevziI4U0d9Ejo0CscTc0YiGsntmGh8SgPl +YHUhwjsj5DznVkyTj2ilDVkXDMdnKH36cd/Ti7rI9xhGwWiWroJs8GZwwltoNKrLZz5hvpGJRjza +N4FRPRfTNV9el2USWM1qSMM1Y/7NRXaaonBN8gJch5DjD0u4v2qFMXnN8/phbfYtvHzjjwT0+iuU +7ZerEj5DV83FPCuPu3neM8vqLO0O2ZdYIhh8N/b+mtVfh1BGmdBeqlwbQzupyi2jtYa8UsRJNryJ +aQNAI1nqE7WkQNi4jnfR3c1mpqrpJm9A0iz0IIK2o3sjIVAY1nB1DvOAql4uV2+L9cSs8YjEiGEo +KG35YpgUoOdQZCSAppLebh6rTHdjmovruZpV28o2pc/Oy336pr4cq5pWJCbrpHOKgF7Zcqw7aT4g +01OxZVG4iurQUBIhovnbHxdfHZBtNKS0ArwffI7sbu7LG2vwJOdvqAnnEc1B+VI9kLqqZRlCNAqE +kJ4j/cKu0mlq4nxv4wxpsPeN6Uf8Lj+9yHoDPxnoy9DeibBGvgTenUCS2KC0hHgbfnGNkmd0fpoi +RQhvUxRoEpPDlfjnWAk8g/IPetdAEKoo9N/xZpmmxIz9M8JFZWhAkmoMc/Gr05iSYBD1gwCfZkRZ +5LJSITBvOi78pRN1fMct5dW8K7bbefN3s+EmS3kmhB12o0t2X08rv7a4nHwnlNQ3SfirkWaQlfL2 +QV7HjU6X5atv072ArDX9fJYxjdG22he+Fk1eYDx63oS7LdXwcETBaDO7z60tQSUp1YSmFEpnocFt +R5dqK/U8dnxUo80Cjm/DerWXkUPtfLpQqIV4DpNrQBTEc5TUlfKNUa61N2QvJXyDME4A4Ynm1vdR +XZJcUAVu03nuxvlUFolusR/Qu0LOPLMqD5pX1cjgEGQkkS9OWjp0YsvQiVVDItpE3k2C4ETWC8MW +wnHFuhVF1UB5o5BC6wy3/wgejsJ84lyMtD1vF3/OZftjIW94ksFUdS0mJ75Jj0wKKdXHclhv9A== diff --git a/images/slurmbase/slurm.conf b/images/slurmbase/slurm.conf new file mode 100755 index 0000000000000000000000000000000000000000..74bb787208e0abe6e94548cb39c1d6b73d52ba79 --- /dev/null +++ b/images/slurmbase/slurm.conf @@ -0,0 +1,183 @@ +# slurm.conf file generated by configurator.html. +# Put this file on all nodes of your cluster. +# See the slurm.conf man page for more information. +# +ControlMachine=slurmclustermaster-main +#ControlAddr= +#BackupController= +#BackupAddr= +# +AuthType=auth/munge +CacheGroups=0 +#CheckpointType=checkpoint/none +CryptoType=crypto/munge +#DisableRootJobs=NO +#EnforcePartLimits=NO +#Epilog= +#EpilogSlurmctld= +#FirstJobId=1 +#MaxJobId=999999 +#GresTypes= +#GroupUpdateForce=0 +#GroupUpdateTime=600 +JobCheckpointDir=/var/lib/slurm-llnl/checkpoint +#JobCredentialPrivateKey= +#JobCredentialPublicCertificate= +#JobFileAppend=0 +#JobRequeue=1 +#JobSubmitPlugins=1 +#KillOnBadExit=0 +#LaunchType=launch/slurm +#Licenses=foo*4,bar +#MailProg=/usr/bin/mail +#MaxJobCount=5000 +#MaxStepCount=40000 +#MaxTasksPerNode=128 +MpiDefault=none +#MpiParams=ports=#-# +#PluginDir= +#PlugStackConfig= +#PrivateData=jobs +ProctrackType=proctrack/pgid +#Prolog= +#PrologSlurmctld= +#PropagatePrioProcess=0 +#PropagateResourceLimits= +#PropagateResourceLimitsExcept= +#RebootProgram= +ReturnToService=1 +#SallocDefaultCommand= +SlurmctldPidFile=/var/run/slurm-llnl/slurmctld.pid +SlurmctldPort=6817 +SlurmdPidFile=/var/run/slurm-llnl/slurmd.pid +SlurmdPort=6818 +SlurmdSpoolDir=/var/lib/slurm-llnl/slurmd +SlurmUser=slurm +#SlurmdUser=root +#SrunEpilog= +#SrunProlog= +StateSaveLocation=/var/lib/slurm-llnl/slurmctld +SwitchType=switch/none +#TaskEpilog= +TaskPlugin=task/none +#TaskPluginParam= +#TaskProlog= +#TopologyPlugin=topology/tree +#TmpFS=/tmp +#TrackWCKey=no +#TreeWidth= +#UnkillableStepProgram= +#UsePAM=0 +# +# +# TIMERS +#BatchStartTimeout=10 +#CompleteWait=0 +#EpilogMsgTime=2000 +#GetEnvTimeout=2 +#HealthCheckInterval=0 +#HealthCheckProgram= +InactiveLimit=0 +KillWait=30 +#MessageTimeout=10 +#ResvOverRun=0 +MinJobAge=300 +#OverTimeLimit=0 +SlurmctldTimeout=120 +SlurmdTimeout=300 +#UnkillableStepTimeout=60 +#VSizeFactor=0 +Waittime=0 +# +# +# SCHEDULING +#DefMemPerCPU=0 +FastSchedule=1 +#MaxMemPerCPU=0 +#SchedulerRootFilter=1 +#SchedulerTimeSlice=30 +SchedulerType=sched/builtin +SchedulerPort=7321 +SelectType=select/linear +#SelectTypeParameters= +# +# +# JOB PRIORITY +#PriorityFlags= +#PriorityType=priority/basic +#PriorityDecayHalfLife= +#PriorityCalcPeriod= +#PriorityFavorSmall= +#PriorityMaxAge= +#PriorityUsageResetPeriod= +#PriorityWeightAge= +#PriorityWeightFairshare= +#PriorityWeightJobSize= +#PriorityWeightPartition= +#PriorityWeightQOS= +# +# +# LOGGING AND ACCOUNTING +#AccountingStorageEnforce=0 +#AccountingStorageHost= +#AccountingStorageLoc= +#AccountingStoragePass= +#AccountingStoragePort= +AccountingStorageType=accounting_storage/none +#AccountingStorageUser= +AccountingStoreJobComment=YES +ClusterName=cluster +#DebugFlags= +#JobCompHost= +JobCompLoc=/var/log/slurm-llnl/jobs.log +#JobCompPass= +#JobCompPort= +JobCompType=jobcomp/filetxt +#JobCompUser= +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/none +SlurmctldDebug=3 +SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log +SlurmdDebug=3 +SlurmdLogFile=/var/log/slurm-llnl/slurmd.log +#SlurmSchedLogFile= +#SlurmSchedLogLevel= +# +# +# POWER SAVE SUPPORT FOR IDLE NODES (optional) +#SuspendProgram= +#ResumeProgram= +#SuspendTimeout= +#ResumeTimeout= +#ResumeRate= +#SuspendExcNodes= +#SuspendExcParts= +#SuspendRate= +#SuspendTime= +# +# Must add controller node explictly but don't place it into any partition +NodeName=slurmclustermaster-main CPUs=1 State=UNKNOWN +#NodeName=partitiona-instrument CPUs=1 State=UNKNOWN +#NodeName=partitionb-instrument CPUs=1 State=UNKNOWN +#NodeName=cris-instrument CPUs=1 State=UNKNOWN +# COMPUTE NODES +NodeName=slurmclusterworker-one CPUs=1 State=UNKNOWN +#NodeName=slurmclusterworker-two CPUs=1 State=UNKNOWN +PartitionName=partition1 Nodes=slurmclusterworker-one MaxTime=INFINITE State=UP +#PartitionName=partition2 Nodes=slurmclusterworker-two MaxTime=INFINITE State=UP + + + + + + + + + + + + + + + + diff --git a/images/slurmbase/supervisord_munge.conf b/images/slurmbase/supervisord_munge.conf new file mode 100755 index 0000000000000000000000000000000000000000..6b8841b47b714550af2f389e82f726808c7a1a3b --- /dev/null +++ b/images/slurmbase/supervisord_munge.conf @@ -0,0 +1,18 @@ +[program:munged] + +; Process definition +process_name = munged +command = /usr/sbin/munged -f --key-file /etc/munge/munge.key -F +autostart = true +autorestart = true +startsecs = 5 +stopwaitsecs = 10 +priority = 100 + +; Log files +stdout_logfile = /var/log/supervisor/%(program_name)s_out.log +stdout_logfile_maxbytes = 100MB +stdout_logfile_backups = 5 +stderr_logfile = /var/log/supervisor/%(program_name)s_err.log +stderr_logfile_maxbytes = 100MB +stderr_logfile_backups = 5 diff --git a/images/slurmcluster/Dockerfile b/images/slurmcluster/Dockerfile new file mode 100755 index 0000000000000000000000000000000000000000..56ce3ccecc0eaac8db543ad82a08935b55b3c938 --- /dev/null +++ b/images/slurmcluster/Dockerfile @@ -0,0 +1,5 @@ +FROM rosetta/slurmbase +MAINTAINER Stefano Alberto Russo <stefano.russo@gmail.com> + +# Add Slurm supervisord conf +COPY supervisord_slurm* /etc/supervisor/conf.d/ diff --git a/images/slurmcluster/supervisord_slurmd.conf b/images/slurmcluster/supervisord_slurmd.conf new file mode 100755 index 0000000000000000000000000000000000000000..e32f9c59f6f6461aec5378564b3eba5d3372ee66 --- /dev/null +++ b/images/slurmcluster/supervisord_slurmd.conf @@ -0,0 +1,18 @@ +[program:slurmd] + +; Process definition +process_name = slurmd +command = /usr/sbin/slurmd -D -f /etc/slurm-llnl/slurm.conf +autostart = true +autorestart = true +startsecs = 5 +stopwaitsecs = 10 +priority = 200 + +; Log files +stdout_logfile = /var/log/supervisor/%(program_name)s_out.log +stdout_logfile_maxbytes = 100MB +stdout_logfile_backups = 5 +stderr_logfile = /var/log/supervisor/%(program_name)s_err.log +stderr_logfile_maxbytes = 100MB +stderr_logfile_backups = 5 diff --git a/images/slurmclustermaster/Dockerfile b/images/slurmclustermaster/Dockerfile new file mode 100755 index 0000000000000000000000000000000000000000..9bdae9ec8e3188915d26ec08e790507d7e02c6ed --- /dev/null +++ b/images/slurmclustermaster/Dockerfile @@ -0,0 +1,11 @@ +FROM rosetta/slurmcluster +MAINTAINER Stefano Alberto Russo <stefano.russo@gmail.com> + +# Configure supervisord to run SLURM +COPY supervisord_slurm* /etc/supervisor/conf.d/ + +# Add sample job script +COPY test.sh /rosetta + +# Add prestartup +COPY prestartup_slurmclustermaster.sh /prestartup/ diff --git a/images/slurmclustermaster/prestartup_slurmclustermaster.sh b/images/slurmclustermaster/prestartup_slurmclustermaster.sh new file mode 100644 index 0000000000000000000000000000000000000000..e1e24db9d1374b11d118d6ec477c6c9977514dc8 --- /dev/null +++ b/images/slurmclustermaster/prestartup_slurmclustermaster.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +mkdir -p /shared/rosetta && chown rosetta:rosetta /shared/rosetta diff --git a/images/slurmclustermaster/supervisord_slurmctld.conf b/images/slurmclustermaster/supervisord_slurmctld.conf new file mode 100755 index 0000000000000000000000000000000000000000..be9514516568efb8cfb858cb6284d5b6f09cf77e --- /dev/null +++ b/images/slurmclustermaster/supervisord_slurmctld.conf @@ -0,0 +1,18 @@ +[program:slurmctld] + +; Process definition +process_name = slurmctld +command = /usr/sbin/slurmctld -D -f /etc/slurm-llnl/slurm.conf +autostart = true +autorestart = true +startsecs = 5 +stopwaitsecs = 10 +priority = 300 + +; Log files +stdout_logfile = /var/log/supervisor/%(program_name)s_out.log +stdout_logfile_maxbytes = 100MB +stdout_logfile_backups = 5 +stderr_logfile = /var/log/supervisor/%(program_name)s_err.log +stderr_logfile_maxbytes = 100MB +stderr_logfile_backups = 5 diff --git a/images/slurmclustermaster/test.sh b/images/slurmclustermaster/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..c8d4c8e7972a70a81d5101136de898f926b6200c --- /dev/null +++ b/images/slurmclustermaster/test.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +#SBATCH --job-name=test +#SBATCH --output=res.txt +#SBATCH --ntasks=1 + +srun hostname +srun sleep 60 \ No newline at end of file diff --git a/images/slurmclusterworker/Dockerfile b/images/slurmclusterworker/Dockerfile new file mode 100755 index 0000000000000000000000000000000000000000..f72bb5e303e8770822e3816e6d8f06215d6e648d --- /dev/null +++ b/images/slurmclusterworker/Dockerfile @@ -0,0 +1,2 @@ +FROM rosetta/slurmcluster +MAINTAINER Stefano Alberto Russo <stefano.russo@gmail.com> diff --git a/rosetta/build b/rosetta/build new file mode 100755 index 0000000000000000000000000000000000000000..347a2c139050aafe99cd531b4674f7876289e4b6 --- /dev/null +++ b/rosetta/build @@ -0,0 +1,44 @@ +#!/bin/bash +set -e + +# Check if we are in the right place +if [ ! -d ./images ]; then + echo "You must run this command from the project's root folder." + exit 1 +fi + + +# Set service and cacheing +if [[ "x$1" == "xnocache" ]] ; then + NOCACHE=True + SERVICE="" +elif [[ "x$2" == "xnocache" ]] ; then + NOCACHE=True + SERVICE=$1 +else + NOCACHE=False + SERVICE=$1 +fi + +if [[ "x$NOCACHE" == "xTrue" ]] ; then + BUILD_COMMAND="docker build --no-cache" +else + BUILD_COMMAND="docker build" +fi + +if [[ "x$SERVICE" == "x" ]] ; then + + # Build all images + $BUILD_COMMAND images/base -t rosetta/base + $BUILD_COMMAND images/slurmbase -t rosetta/slurmbase + $BUILD_COMMAND images/slurmcluster -t rosetta/slurmcluster + $BUILD_COMMAND images/slurmclustermaster -t rosetta/slurmclustermaster + $BUILD_COMMAND images/slurmclusterworker -t rosetta/slurmclusterworker + $BUILD_COMMAND images/dregistry -t rosetta/dregistry + +else + + # Build a specific image + $BUILD_COMMAND images/$SERVICE -t rosetta/$SERVICE + +fi diff --git a/rosetta/clean b/rosetta/clean new file mode 100755 index 0000000000000000000000000000000000000000..10e582aadf0e268da768d9b41dadec9a66d7a4ac --- /dev/null +++ b/rosetta/clean @@ -0,0 +1,13 @@ +#!/bin/bash + +# Check if we are in the right place +if [ ! -d ./images ]; then + echo "You must run this command from the project's root folder." + exit 1 +fi + +if [[ $# -eq 0 ]] ; then + docker-compose down +else + docker-compose down $@ +fi diff --git a/rosetta/ps b/rosetta/ps new file mode 100755 index 0000000000000000000000000000000000000000..0e2f7bfda0ce73fbc8c732e5726b6c6cbde13153 --- /dev/null +++ b/rosetta/ps @@ -0,0 +1,14 @@ +#!/bin/bash + +# Check if we are in the right place +if [ ! -d ./images ]; then + echo "You must run this command from the project's root folder." + exit 1 +fi + +if [[ $# -eq 0 ]] ; then + docker-compose ps +else + echo "This command does not support any argument." + exit 1 +fi diff --git a/rosetta/rerun b/rosetta/rerun new file mode 100755 index 0000000000000000000000000000000000000000..2ceb7540912cc7cecbd9e05da284b5443137f5b9 --- /dev/null +++ b/rosetta/rerun @@ -0,0 +1,14 @@ +#!/bin/bash + +# Check if we are in the right place +if [ ! -d ./images ]; then + echo "You must run this command from the project's root folder." + exit 1 +fi + +if [[ $# -eq 0 ]] ; then + docker-compose down + docker-compose up -d +else + docker-compose up -d --no-deps $@ +fi diff --git a/rosetta/run b/rosetta/run new file mode 100755 index 0000000000000000000000000000000000000000..2ae356c8ba92bce1bae66c171932b9f0f4000da2 --- /dev/null +++ b/rosetta/run @@ -0,0 +1,13 @@ +#!/bin/bash + +# Check if we are in the right place +if [ ! -d ./images ]; then + echo "You must run this command from the project's root folder." + exit 1 +fi + +if [[ $# -eq 0 ]] ; then + docker-compose up -d +else + docker-compose up $@ -d +fi diff --git a/rosetta/shell b/rosetta/shell new file mode 100755 index 0000000000000000000000000000000000000000..deda8d7f4dbeae0d60c7db699f71a6a6a097b495 --- /dev/null +++ b/rosetta/shell @@ -0,0 +1,31 @@ +#!/bin/bash + +# Check if we are in the right place +if [ ! -d ./images ]; then + echo "You must run this command from the project's root folder." + exit 1 +fi + +if [[ $# -eq 0 ]] ; then + echo "Please tell me on which service to open the shell in." + exit 1 + +elif [[ $# -gt 2 ]] ; then + echo "Use double quotes to wrap commands with spaces" + exit 1 +else + + COMMAND=$2 + if [[ "x$COMMAND" == "x" ]] ; then + echo "" + echo "Executing: /bin/bash" + echo "" + docker-compose exec $1 sudo -i -u rosetta /bin/bash + else + echo "" + echo "Executing: \"$COMMAND\"" + echo "" + docker-compose exec $1 sudo -i -u rosetta /bin/bash -c "$COMMAND" + fi + +fi diff --git a/rosetta/status b/rosetta/status new file mode 100755 index 0000000000000000000000000000000000000000..ecd91be98a1d508195a434b82ddcd46fb30cf36e --- /dev/null +++ b/rosetta/status @@ -0,0 +1,33 @@ +#!/bin/bash + +# Check if we are in the right place +if [ ! -d ./images ]; then + echo "You must run this command from the project's root folder." + exit 1 +fi + +if [[ $# -eq 0 ]] ; then + + declare -a container_names + OUT=$(rosetta/ps) + + while read -r line; do + + if [[ $line == *"Up"* ]]; then + container_name=$(echo $line | cut -d ' ' -f1) + container_names+=($container_name); + fi + + done <<< "$OUT" + + for container_name in ${container_names[@]} + do + echo "" + echo "$container_name" + docker-compose exec $container_name /bin/bash -c "supervisorctl status" + done + echo "" + +else + docker-compose exec $@ /bin/bash -c "supervisorctl status" +fi