Updated version of DDFacet pipeline

This container employ:
- lofar-release-4_0_16
- ddfacet branch ssd-py3_kms

ddf2/Dockerfile

-FROM lofarit/base3.10
-
-USER root
-
-RUN apt-get install llvm-7 -y
-ENV LLVM_CONFIG=llvm-config-7 
-ENV PYTHONPATH /opt/lofarsoft/lib/python2.7/site-packages
-#------------------------
-# Get and install ddf2
-#------------------------
-# DDF pipeline
-RUN cd /opt && git clone https://github.com/mhardcastle/ddf-pipeline.git
-RUN cd /opt/ddf-pipeline && git checkout fdaa5aa
-
-# SkyModel
-RUN cd /opt && git clone https://github.com/cyriltasse/SkyModel.git
-RUN cd /opt/SkyModel && git checkout 026997f
-
-# killMS
-COPY killMS /opt/killMS
-#RUN cd /opt/killMS && python setup.py build
-RUN cd /opt/killMS/Predict && make
-RUN cd /opt/killMS/Array/Dot && make
-RUN cd /opt/killMS/Gridder && make
-
-# DynSpecMS
-RUN cd /opt && git clone https://github.com/cyriltasse/DynSpecMS.git
-RUN cd /opt/DynSpecMS && git checkout 461183f
-
-# DDFacet
-COPY DDFacet /opt/DDFacet
-RUN cp /opt/ddf-pipeline/misc/setup.cfg /opt/DDFacet/ # Fix compile options
-RUN python -m pip install -U pip setuptools wheel
-RUN pip install -U pip setuptools wheel
-RUN python -m pip install numpy==1.16.2 bdsf==1.8.15 emcee
-
-RUN python -m pip install astropy_healpix pybind11 future pyregion sshtunnel pymysql psutil
-RUN python -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]"
-
-RUN cd /opt/DDFacet && rm -rf /opt/DDFacet/Dcbuild && python setup.py build
-
-RUN python -m pip install bdsf==1.8.15 numpy==1.16.2
-RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/init.sh
+FROM lofar/lofar-pipeline:LOFAR-Release-4_0_16
 
-RUN  sed 's/numpy (<=1.16)/numpy (<=1.16.2)/g' /usr/local/lib/python2.7/dist-packages/meqtrees_cattery-1.7.0.dist-info/METADATA >  /usr/local/lib/python2.7/dist-packages/meqtrees_cattery-1.7.0.dist-info/METADATA
+#Package dependencies
+COPY apt.sources.list /etc/apt/sources.list
 
-RUN cd /opt && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git
+# Support large mlocks
+RUN echo "*        -   memlock     unlimited" > /etc/security/limits.conf
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBIAN_PRIORITY critical
+ENV GNUCOMPILER 7
+ENV DEB_SETUP_DEPENDENCIES \
+    dpkg-dev \
+    g++-$GNUCOMPILER \
+    gcc-$GNUCOMPILER \
+    libc-dev \
+    cmake \
+    gfortran-$GNUCOMPILER \
+    git \
+    wget \
+    subversion
+
+ENV DEB_DEPENCENDIES \
+    python3-pip \
+    libfftw3-dev \
+    python3-numpy \
+    libfreetype6 \
+    libfreetype6-dev \
+    libpng-dev \
+    pkg-config \
+    python3-dev \
+    libboost-all-dev \
+    libcfitsio-dev \
+    libhdf5-dev \
+    wcslib-dev \
+    libatlas-base-dev \
+    liblapack-dev \
+    python3-tk \
+    libreadline6-dev \
+    subversion \
+    liblog4cplus-dev \
+    libhdf5-dev \
+    libncurses5-dev \
+    libsofa1-dev \
+    flex \
+    bison \
+    libbison-dev \
+    # Reference image generation dependencies
+    make
+
+RUN apt-get update
+RUN apt-get install -y $DEB_SETUP_DEPENDENCIES
+RUN apt-get install -y $DEB_DEPENCENDIES
+
+ENV PATH /usr/local/bin:$PATH
+ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
+ENV PYTHONPATH /usr/local/lib/python2.7/site-packages:$PYTHONPATH
+
+# Latest GCC segfaults when compiling casacore
+RUN rm /usr/bin/gcc /usr/bin/g++ /usr/bin/cpp /usr/bin/cc
+RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/gcc
+RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/g++
+RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/cc
+RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/cpp
+RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran
+
+
+
+#####################################################################
+## BUILD DDF FROM SOURCE
+#####################################################################
+#Copy DDFacet and SkyModel into the image
+ADD DDFacet/DDFacet /opt/DDFacet/DDFacet
+ADD DDFacet/SkyModel /opt/DDFacet/SkyModel
+ADD DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in
+ADD DDFacet/setup.py /opt/DDFacet/setup.py
+ADD DDFacet/setup.cfg /opt/DDFacet/setup.cfg
+ADD DDFacet/README.rst /opt/DDFacet/README.rst
+ADD DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml
+ADD DDFacet/.git /opt/DDFacet/.git
+ADD DDFacet/.gitignore /opt/DDFacet/.gitignore
+ADD DDFacet/.gitmodules /opt/DDFacet/.gitmodules
 
-#USER lofar
 
-ENV CONTAINER_NAME='ddf2_base3.10'
+# Finally install DDFacet
+RUN rm -rf /opt/DDFacet/DDFacet/cbuild
+RUN pip3 install -U pip setuptools wheel
+RUN python3 -m pip install pybind11
+RUN python3 -m pip install tensorflow==1.8.0
+RUN python3 -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]"
+RUN cd /opt/DDFacet/ && python3 setup.py build && cd /
+# Set MeqTrees Cattery path to installation directory
+ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/
+ENV PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH
+
+RUN python3 -m pip install numpy==1.17.0
+
+
+RUN python3 -c "import Siamese"
+RUN python3 -c "import bdsf"
+
+# perform some basic tests
+RUN DDF.py --help
+RUN MakeMask.py --help
+RUN MakeCatalog.py --help
+RUN MakeModel.py --help
+RUN MaskDicoModel.py --help
+RUN ClusterCat.py --help
+
+#####################################################################
+## killMs
+#####################################################################
+ADD killMS /opt/killMS
+RUN pip3 install /opt/killMS
+RUN cd  /opt/killMS; python3 setup.py build
+
+#####################################################################
+## DDF-PIPELINE
+#####################################################################
+RUN pip3 install pyregion sshtunnel pymysql
+RUN  cd /opt; git clone https://github.com/mhardcastle/ddf-pipeline
+RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/init.sh
+
+RUN cd /opt && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git
+
 ENV DDF_PIPELINE_CATALOGS='/opt/container-data/bootstrap-cats/'
+ENV CONTAINER_NAME='ddfacet_ssd-py3_kms-lofar-release-4_0_16'
\ No newline at end of file

ddf2/apt.sources.list

+###### Ubuntu Main Repos
+deb http://za.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse
+deb-src http://za.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse
+###### Ubuntu Update Repos
+deb http://za.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse
+deb http://za.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse
+deb http://za.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse
+deb http://za.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse
+deb-src http://za.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse
+deb-src http://za.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse
+deb-src http://za.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse
+deb-src http://za.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse

ddf2/build_docker_container.sh

 #!/bin/bash
 set -e 
 rm -rf DDFacet
+rm -rf killMS
+
+git clone https://github.com/cyriltasse/killMS.git
+git checkout 0607798
+
 
 git clone https://github.com/cyriltasse/DDFacet.git
-git checkout 1059028
+git checkout e98968d
+
 
-docker build  . -t lofarit/ddf2_base3.10 
+docker build  . -t lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16