diff --git a/ddf2/Dockerfile b/ddf2/Dockerfile index 8d7a5891eac9b623cd149427066c42c8b12bc63b..d6613101326f567b69fc2e371d8258df62b417c9 100644 --- a/ddf2/Dockerfile +++ b/ddf2/Dockerfile @@ -1,52 +1,127 @@ -FROM lofarit/base3.10 - -USER root - -RUN apt-get install llvm-7 -y -ENV LLVM_CONFIG=llvm-config-7 -ENV PYTHONPATH /opt/lofarsoft/lib/python2.7/site-packages -#------------------------ -# Get and install ddf2 -#------------------------ -# DDF pipeline -RUN cd /opt && git clone https://github.com/mhardcastle/ddf-pipeline.git -RUN cd /opt/ddf-pipeline && git checkout fdaa5aa - -# SkyModel -RUN cd /opt && git clone https://github.com/cyriltasse/SkyModel.git -RUN cd /opt/SkyModel && git checkout 026997f - -# killMS -COPY killMS /opt/killMS -#RUN cd /opt/killMS && python setup.py build -RUN cd /opt/killMS/Predict && make -RUN cd /opt/killMS/Array/Dot && make -RUN cd /opt/killMS/Gridder && make - -# DynSpecMS -RUN cd /opt && git clone https://github.com/cyriltasse/DynSpecMS.git -RUN cd /opt/DynSpecMS && git checkout 461183f - -# DDFacet -COPY DDFacet /opt/DDFacet -RUN cp /opt/ddf-pipeline/misc/setup.cfg /opt/DDFacet/ # Fix compile options -RUN python -m pip install -U pip setuptools wheel -RUN pip install -U pip setuptools wheel -RUN python -m pip install numpy==1.16.2 bdsf==1.8.15 emcee - -RUN python -m pip install astropy_healpix pybind11 future pyregion sshtunnel pymysql psutil -RUN python -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]" - -RUN cd /opt/DDFacet && rm -rf /opt/DDFacet/Dcbuild && python setup.py build - -RUN python -m pip install bdsf==1.8.15 numpy==1.16.2 -RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/init.sh +FROM lofar/lofar-pipeline:LOFAR-Release-4_0_16 -RUN sed 's/numpy (<=1.16)/numpy (<=1.16.2)/g' /usr/local/lib/python2.7/dist-packages/meqtrees_cattery-1.7.0.dist-info/METADATA > /usr/local/lib/python2.7/dist-packages/meqtrees_cattery-1.7.0.dist-info/METADATA +#Package dependencies +COPY apt.sources.list /etc/apt/sources.list -RUN cd /opt && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git +# Support large mlocks +RUN echo "* - memlock unlimited" > /etc/security/limits.conf + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_PRIORITY critical +ENV GNUCOMPILER 7 +ENV DEB_SETUP_DEPENDENCIES \ + dpkg-dev \ + g++-$GNUCOMPILER \ + gcc-$GNUCOMPILER \ + libc-dev \ + cmake \ + gfortran-$GNUCOMPILER \ + git \ + wget \ + subversion + +ENV DEB_DEPENCENDIES \ + python3-pip \ + libfftw3-dev \ + python3-numpy \ + libfreetype6 \ + libfreetype6-dev \ + libpng-dev \ + pkg-config \ + python3-dev \ + libboost-all-dev \ + libcfitsio-dev \ + libhdf5-dev \ + wcslib-dev \ + libatlas-base-dev \ + liblapack-dev \ + python3-tk \ + libreadline6-dev \ + subversion \ + liblog4cplus-dev \ + libhdf5-dev \ + libncurses5-dev \ + libsofa1-dev \ + flex \ + bison \ + libbison-dev \ + # Reference image generation dependencies + make + +RUN apt-get update +RUN apt-get install -y $DEB_SETUP_DEPENDENCIES +RUN apt-get install -y $DEB_DEPENCENDIES + +ENV PATH /usr/local/bin:$PATH +ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH +ENV PYTHONPATH /usr/local/lib/python2.7/site-packages:$PYTHONPATH + +# Latest GCC segfaults when compiling casacore +RUN rm /usr/bin/gcc /usr/bin/g++ /usr/bin/cpp /usr/bin/cc +RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/gcc +RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/g++ +RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/cc +RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/cpp +RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran + + + +##################################################################### +## BUILD DDF FROM SOURCE +##################################################################### +#Copy DDFacet and SkyModel into the image +ADD DDFacet/DDFacet /opt/DDFacet/DDFacet +ADD DDFacet/SkyModel /opt/DDFacet/SkyModel +ADD DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in +ADD DDFacet/setup.py /opt/DDFacet/setup.py +ADD DDFacet/setup.cfg /opt/DDFacet/setup.cfg +ADD DDFacet/README.rst /opt/DDFacet/README.rst +ADD DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml +ADD DDFacet/.git /opt/DDFacet/.git +ADD DDFacet/.gitignore /opt/DDFacet/.gitignore +ADD DDFacet/.gitmodules /opt/DDFacet/.gitmodules -#USER lofar -ENV CONTAINER_NAME='ddf2_base3.10' +# Finally install DDFacet +RUN rm -rf /opt/DDFacet/DDFacet/cbuild +RUN pip3 install -U pip setuptools wheel +RUN python3 -m pip install pybind11 +RUN python3 -m pip install tensorflow==1.8.0 +RUN python3 -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]" +RUN cd /opt/DDFacet/ && python3 setup.py build && cd / +# Set MeqTrees Cattery path to installation directory +ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/ +ENV PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH + +RUN python3 -m pip install numpy==1.17.0 + + +RUN python3 -c "import Siamese" +RUN python3 -c "import bdsf" + +# perform some basic tests +RUN DDF.py --help +RUN MakeMask.py --help +RUN MakeCatalog.py --help +RUN MakeModel.py --help +RUN MaskDicoModel.py --help +RUN ClusterCat.py --help + +##################################################################### +## killMs +##################################################################### +ADD killMS /opt/killMS +RUN pip3 install /opt/killMS +RUN cd /opt/killMS; python3 setup.py build + +##################################################################### +## DDF-PIPELINE +##################################################################### +RUN pip3 install pyregion sshtunnel pymysql +RUN cd /opt; git clone https://github.com/mhardcastle/ddf-pipeline +RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/init.sh + +RUN cd /opt && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git + ENV DDF_PIPELINE_CATALOGS='/opt/container-data/bootstrap-cats/' +ENV CONTAINER_NAME='ddfacet_ssd-py3_kms-lofar-release-4_0_16' \ No newline at end of file diff --git a/ddf2/apt.sources.list b/ddf2/apt.sources.list new file mode 100644 index 0000000000000000000000000000000000000000..b81693dfa2b9e860103e1786a1f6cb871f2b1afd --- /dev/null +++ b/ddf2/apt.sources.list @@ -0,0 +1,12 @@ +###### Ubuntu Main Repos +deb http://za.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse +deb-src http://za.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse +###### Ubuntu Update Repos +deb http://za.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse +deb http://za.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse +deb http://za.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse +deb http://za.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse +deb-src http://za.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse +deb-src http://za.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse +deb-src http://za.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse +deb-src http://za.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse diff --git a/ddf2/build_docker_container.sh b/ddf2/build_docker_container.sh index 7c8ffc5cfbd447e635c8846a42858f9ce09f95ef..d4b0b170f2558a3dba4b26614766d079886a9cf8 100755 --- a/ddf2/build_docker_container.sh +++ b/ddf2/build_docker_container.sh @@ -1,8 +1,14 @@ #!/bin/bash set -e rm -rf DDFacet +rm -rf killMS + +git clone https://github.com/cyriltasse/killMS.git +git checkout 0607798 + git clone https://github.com/cyriltasse/DDFacet.git -git checkout 1059028 +git checkout e98968d + -docker build . -t lofarit/ddf2_base3.10 +docker build . -t lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16