From 15a1414d4fe5402f6ebbe0db8ed59143396408de Mon Sep 17 00:00:00 2001 From: Stefano Alberto Russo <stefano.russo@gmail.com> Date: Fri, 24 Jul 2020 14:53:39 +0200 Subject: [PATCH] Modified ddf2 container to use (external) deployment keys. Minor Fixes. --- .gitignore | 4 +- ddf2/Dockerfile | 80 ++++++++++++++++++++++++++-------- ddf2/build_docker_container.sh | 11 ----- ddf2/run_docker_container.sh | 2 +- 4 files changed, 63 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index c0ff1e7..2fb2dda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1 @@ - -ddf2/DDFacet/ -ddf2/killMS/ +ddf2/keys diff --git a/ddf2/Dockerfile b/ddf2/Dockerfile index d661310..c5eea55 100644 --- a/ddf2/Dockerfile +++ b/ddf2/Dockerfile @@ -1,6 +1,6 @@ FROM lofar/lofar-pipeline:LOFAR-Release-4_0_16 -#Package dependencies +# Package dependencies COPY apt.sources.list /etc/apt/sources.list # Support large mlocks @@ -65,21 +65,48 @@ RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/cpp RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran +##################################################################### +# Add deployment keys +##################################################################### + +RUN mkdir -p /root/.ssh/ +COPY keys/lofarit_ddf2_rsa /root/.ssh/lofarit_ddf2_rsa +COPY keys/lofarit_killms_rsa /root/.ssh/lofarit_killms_rsa +RUN chmod 600 /root/.ssh/lofarit_ddf2_rsa && chmod 600 /root/.ssh/lofarit_killms_rsa + ##################################################################### -## BUILD DDF FROM SOURCE +# Get DDF from private repo ##################################################################### -#Copy DDFacet and SkyModel into the image -ADD DDFacet/DDFacet /opt/DDFacet/DDFacet -ADD DDFacet/SkyModel /opt/DDFacet/SkyModel -ADD DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in -ADD DDFacet/setup.py /opt/DDFacet/setup.py -ADD DDFacet/setup.cfg /opt/DDFacet/setup.cfg -ADD DDFacet/README.rst /opt/DDFacet/README.rst -ADD DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml -ADD DDFacet/.git /opt/DDFacet/.git -ADD DDFacet/.gitignore /opt/DDFacet/.gitignore -ADD DDFacet/.gitmodules /opt/DDFacet/.gitmodules + +# Add github key(s) +RUN ssh-keyscan github.com >> ~/.ssh/known_hosts + +# Clone +RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_ddf2_rsa && cd /root/ && git clone git@github.com:cyriltasse/DDFacet' + +# Checkout +RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_ddf2_rsa && cd /root/DDFacet && git pull && git checkout e98968d' + + + +##################################################################### +# Build DDF from source +##################################################################### + +RUN mkdir /opt/DDFacet + +# Copy DDFacet and SkyModel into the image +RUN cp -a /root/DDFacet/DDFacet /opt/DDFacet/DDFacet && \ + cp -a /root/DDFacet/SkyModel /opt/DDFacet/SkyModel && \ + cp -a /root/DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in && \ + cp -a /root/DDFacet/setup.py /opt/DDFacet/setup.py && \ + cp -a /root/DDFacet/setup.cfg /opt/DDFacet/setup.cfg && \ + cp -a /root/DDFacet/README.rst /opt/DDFacet/README.rst && \ + cp -a /root/DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml && \ + cp -a /root/DDFacet/.git /opt/DDFacet/.git && \ + cp -a /root/DDFacet/.gitignore /opt/DDFacet/.gitignore && \ + cp -a /root/DDFacet/.gitmodules /opt/DDFacet/.gitmodules # Finally install DDFacet @@ -89,17 +116,17 @@ RUN python3 -m pip install pybind11 RUN python3 -m pip install tensorflow==1.8.0 RUN python3 -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]" RUN cd /opt/DDFacet/ && python3 setup.py build && cd / + # Set MeqTrees Cattery path to installation directory ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/ ENV PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH RUN python3 -m pip install numpy==1.17.0 - RUN python3 -c "import Siamese" RUN python3 -c "import bdsf" -# perform some basic tests +# Perform some basic tests RUN DDF.py --help RUN MakeMask.py --help RUN MakeCatalog.py --help @@ -107,15 +134,30 @@ RUN MakeModel.py --help RUN MaskDicoModel.py --help RUN ClusterCat.py --help + ##################################################################### -## killMs +# Get and build killMs ##################################################################### -ADD killMS /opt/killMS + +# Clone +RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_killms_rsa && cd /opt/ && git clone git@github.com:cyriltasse/killMS' + +# Checkout +RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_killms_rsa && cd /opt/killMS && git pull && git checkout 0607798' + RUN pip3 install /opt/killMS RUN cd /opt/killMS; python3 setup.py build + +##################################################################### +# Remove deploymnet keys +##################################################################### + +RUN rm -f /root/.ssh/lofarit_ddf2_rsa && rm -f /root/.ssh/lofarit_killms_rsa + + ##################################################################### -## DDF-PIPELINE +# DDF-PIPELINE ##################################################################### RUN pip3 install pyregion sshtunnel pymysql RUN cd /opt; git clone https://github.com/mhardcastle/ddf-pipeline @@ -124,4 +166,4 @@ RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/ini RUN cd /opt && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git ENV DDF_PIPELINE_CATALOGS='/opt/container-data/bootstrap-cats/' -ENV CONTAINER_NAME='ddfacet_ssd-py3_kms-lofar-release-4_0_16' \ No newline at end of file +ENV CONTAINER_NAME='ddfacet_ssd-py3_kms-lofar-release-4_0_16' diff --git a/ddf2/build_docker_container.sh b/ddf2/build_docker_container.sh index d4b0b17..3b2af75 100755 --- a/ddf2/build_docker_container.sh +++ b/ddf2/build_docker_container.sh @@ -1,14 +1,3 @@ #!/bin/bash -set -e -rm -rf DDFacet -rm -rf killMS - -git clone https://github.com/cyriltasse/killMS.git -git checkout 0607798 - - -git clone https://github.com/cyriltasse/DDFacet.git -git checkout e98968d - docker build . -t lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16 diff --git a/ddf2/run_docker_container.sh b/ddf2/run_docker_container.sh index 951d413..fa3e190 100755 --- a/ddf2/run_docker_container.sh +++ b/ddf2/run_docker_container.sh @@ -1,2 +1,2 @@ #!/bin/bash -docker run --rm -v $PWD/data:/data -it lofarit/ddf2_base3.10 /bin/bash +docker run --rm -v $PWD/data:/data -it lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16 /bin/bash -- GitLab