Skip to content
Snippets Groups Projects
Commit e865ac22 authored by Giacomo Mulas's avatar Giacomo Mulas
Browse files

Merge branch 'containers' into 'master'

create new Dockerfile to create docker images both for the pipeline runner and...

See merge request giacomo.mulas/np_tmcode!15
parents d08c4c24 a1b520dc
No related branches found
No related tags found
No related merge requests found
...@@ -3,4 +3,3 @@ build/cluster/* ...@@ -3,4 +3,3 @@ build/cluster/*
build/sphere/* build/sphere/*
build/trapping/* build/trapping/*
doc/build/* doc/build/*
...@@ -30,7 +30,8 @@ building_stage: ...@@ -30,7 +30,8 @@ building_stage:
paths: paths:
- build/cluster/* - build/cluster/*
- build/sphere/* - build/sphere/*
#- build/trapping/* - build/trapping/*
- doc/build/*
exclude: exclude:
- ".git*" - ".git*"
- ".git/**/*" - ".git/**/*"
...@@ -45,7 +46,8 @@ building_stage: ...@@ -45,7 +46,8 @@ building_stage:
- cat /etc/os-release - cat /etc/os-release
- cd src - cd src
- echo "Running make ..." - echo "Running make ..."
- make - make -j
- make docs -j && make -C ../doc/build/latex -j
running_stage: running_stage:
stage: run stage: run
......
...@@ -13,15 +13,15 @@ This section describes the use of the pre-existing programs, once the binaries h ...@@ -13,15 +13,15 @@ This section describes the use of the pre-existing programs, once the binaries h
### cluster ### cluster
1. cd to the `build/cluster` folder. 1. cd to the `build/cluster` folder.
2. Run `edfb`: 2. Run `edfb_clu`:
> ./edfb > ./edfb_clu
3. Run `clu`: 3. Run `clu`:
> ./clu > ./clu
*NOTE:* both `edfb` and `clu` expect an input which is assumed to be in a folder named `../../test_data/cluster/` (i.e. two levels above the current execution path) *NOTE:* both `edfb_clu` and `clu` expect an input which is assumed to be in a folder named `../../test_data/cluster/` (i.e. two levels above the current execution path)
4. Run `np_cluster`: 4. Run `np_cluster`:
...@@ -31,22 +31,22 @@ This section describes the use of the pre-existing programs, once the binaries h ...@@ -31,22 +31,22 @@ This section describes the use of the pre-existing programs, once the binaries h
5. Check the consistency between the output files (the default output file for the *FORTRAN* code is named `OCLU`, while the corresponding *C++* output has the default name of `c_OCLU`). 5. Check the consistency between the output files (the default output file for the *FORTRAN* code is named `OCLU`, while the corresponding *C++* output has the default name of `c_OCLU`).
The default behaviour of `np_cluster` is to take the same input files as `edfb` and `clu` and to write the output in the current folder. If needed, different input and output paths can be given as command-line arguments: The default behaviour of `np_cluster` is to take the same input files as `edfb_clu` and `clu` and to write the output in the current folder. If needed, different input and output paths can be given as command-line arguments:
> ./np_cluster PATH_TO_DEDFB PATH_TO_DCLU OUTPUT_PATH > ./np_cluster PATH_TO_DEDFB PATH_TO_DCLU OUTPUT_PATH
### sphere ### sphere
1. cd to the `build/sphere` folder. 1. cd to the `build/sphere` folder.
2. Run `edfb`: 2. Run `edfb_sph`:
> ./edfb > ./edfb_sph
3. Run `sph`: 3. Run `sph`:
> ./sph > ./sph
*NOTE:* both `edfb` and `sph` expect an input which is assumed to be in a folder named `../../test_data/sphere/` (i.e. two levels above the current execution path) *NOTE:* both `edfb_sph` and `sph` expect an input which is assumed to be in a folder named `../../test_data/sphere/` (i.e. two levels above the current execution path)
4. Run `np_sphere`: 4. Run `np_sphere`:
...@@ -56,7 +56,7 @@ The default behaviour of `np_cluster` is to take the same input files as `edfb` ...@@ -56,7 +56,7 @@ The default behaviour of `np_cluster` is to take the same input files as `edfb`
5. Check the consistency between the output files (the default output file for the *FORTRAN* code is named `OSPH`, while the corresponding *C++* output has the default name of `c_OSPH`). 5. Check the consistency between the output files (the default output file for the *FORTRAN* code is named `OSPH`, while the corresponding *C++* output has the default name of `c_OSPH`).
The default behaviour of `np_sphere` is to take the same input files as `edfb` and `sph` and to write the output in the current folder. If needed, different input and output paths can be given as command-line arguments: The default behaviour of `np_sphere` is to take the same input files as `edfb_sph` and `sph` and to write the output in the current folder. If needed, different input and output paths can be given as command-line arguments:
> ./np_sphere PATH_TO_DEDFB PATH_TO_DSPH OUTPUT_PATH > ./np_sphere PATH_TO_DEDFB PATH_TO_DSPH OUTPUT_PATH
......
# syntax=docker/dockerfile:1
FROM debian:bookworm-slim AS np-tmcode-dev
WORKDIR /root
# make sure the debian bullseye us up to date, install needed packages
RUN DEBIAN_FRONTEND=noninteractive apt update && DEBIAN_FRONTEND=noninteractive apt -y upgrade
# install packages needed to build binaries
RUN DEBIAN_FRONTEND=noninteractive apt -y install g++ gfortran make gcc-offload-nvptx libhdf5-dev
# install packages needed to run python scripts for checks
RUN DEBIAN_FRONTEND=noninteractive apt -y install python3 python-is-python3 python3-regex
# install packages needed to run doxygen to create html docs
RUN DEBIAN_FRONTEND=noninteractive apt -y install doxygen
# install packages needed to run pdflatex to create pdf docs
RUN DEBIAN_FRONTEND=noninteractive apt -y install texlive-latex-base texlive-latex-recommended texlive-latex-extra texlive-font-utils
# clean up apt lists
RUN rm -rf /var/lib/apt/lists/*
# create the container on which the gitlab pipeline runs, making sure to squash
# all layers to save space
FROM scratch AS np-tmcode
COPY --from=np-tmcode-dev / /
# create the container actually containing the code
FROM np-tmcode-dev AS np-tmcode-run-dev
WORKDIR /root
# copy the parent dir to a temporary dir in the container, so we can compile everything
ADD src /root/np-tmcode/src
ADD doc /root/np-tmcode/doc
ADD build /root/np-tmcode/build
ADD test_data /root/np-tmcode/test_data
RUN cd np-tmcode/src && make wipe && make -j && cd ../doc/src && doxygen config.dox && cd ../build/latex && make -j
# create the container on which the np-tmcode is installed, restarting from
# a minimal distro and adding strictly only the needed runtime
FROM debian:bookworm-slim AS np-tmcode-run-minimal
WORKDIR /root
# install the strictly needed runtime libraries needed to run the executables
# and the python check scripts
RUN DEBIAN_FRONTEND=noninteractive apt update && DEBIAN_FRONTEND=noninteractive apt upgrade && DEBIAN_FRONTEND=noninteractive apt -y install libgfortran5 libgcc-s1 libhdf5-103-1 libstdc++6 libssl3 libcurl4 libsz2 zlib1g libnghttp2-14 libidn2-0 librtmp1 libssh2-1 libpsl5 libgssapi-krb5-2 libldap-2.5-0 libzstd1 libbrotli1 libaec0 libunistring2 libgmp10 libkrb5-3 libk5crypto3 libcom-err2 libkrb5support0 libsasl2-2 libp11-kit0 libtasn1-6 libkeyutils1 libffi8 python3 python-is-python3 python3-regex hdf5-tools && rm -rf /var/lib/apt/lists/*
COPY --from=np-tmcode-run-dev /root /root
# remove everything which is not needed to run the codes
RUN cd /root/np-tmcode && find build -name "*.o" -exec rm -v \{\} \; && find build -name "*.gcno" -exec rm -v \{\} \; && cd src && rm -rvf cluster libnptm trapping include sphere Makefile make.inc README.md && cd .. && rm -rvf containers && cd doc && rm -rvf src && cd build/latex && rm -rvf *.tex *.out *.sty *.ind *.log *.toc *.ilg *.idx *.aux *.eps Makefile class*.pdf
# move the installed software to /usr/local
RUN mv /root/np-tmcode /usr/local
# create the container on which the np-tmcode is installed,
# squashing all layers from previous stage
FROM scratch AS np-tmcode-run
WORKDIR /root
COPY --from=np-tmcode-run-minimal / /
# Docker support for the NP-TMcode project
This Dockerfile allows to create mainly:
- the np-tmcode image that is used by the gitlab pipeline runner to test compile and execution of requested merges
- the np-tmcode-run image, that contains only the pre-built executables, python test scripts, compiled documentation, and the minimal runtime to run them
In order for the docker build to correctly set the context, so that it can access the directories containing source codes, docker build must be invoked from this directory as:
> docker build -f Dockerfile ../.. --target \<image name\>
where `<image name>` is either `np-tmcode` or `np-tmcode-run`. One may also add the docker build option `-t tagname` to specify the `tagname` the built image should have in his/her registry.
\ No newline at end of file
# Singularity support for the NP-TMcode project
This `np-tmcode-run.def` file allows to create a np-tmcode-run image, that contains only the pre-built executables, python test scripts, compiled documentation, and the minimal runtime to run them
In order for the singularity build to create the image, singularity build must be invoked from this directory as:
> singularity build \<image name\> np-tmcode-run.def
where `<image name>` is the name one wants to give to the image, e.g. `np-tmcode-run.sif`. Depending on the singularity installation, the additional `--fakeroot` option may be needed (see singularity documentation).
The image thus built can be run either as
> singularity run \<image name\> \<np-tmcode command\>
or directly as
> \<full path to image name\> \<np-tmcode command\>
where `<full path to image name>` is the name of the sif image, including full or relative path to it, and `<np-tmcode command>` is one of the executables built in np-tmcode project, or the `pycompare.py` script used to compare results between the Fortran and C++ versions of the code. These executables, as well as the minimum runtime for them to execute correctly, are embedded in the image, and "see" the current directory the image is run in (which technically is automatically bind-mounted by singularity in the running image instance).
\ No newline at end of file
Bootstrap: docker
From: debian:bookworm-slim
Stage: np-tmcode-run-dev
%files
../../src /usr/local/np-tmcode/src
../../doc /usr/local/np-tmcode/doc
../../build /usr/local/np-tmcode/build
../../test_data /usr/local/np-tmcode/test_data
%post
apt update
apt -y upgrade
apt -y install g++ gfortran make gcc-offload-nvptx libhdf5-dev
apt -y install python3 python-is-python3 python3-regex
apt -y install doxygen
apt -y install texlive-latex-base texlive-latex-recommended texlive-latex-extra texlive-font-utils
rm -rf /var/lib/apt/lists/*
cd /usr/local/np-tmcode/src
make wipe
make -j
make docs -j && make -C ../doc/build/latex -j
Bootstrap: docker
From: debian:bookworm-slim
Stage: np-tmcode-run-minimal
%files from np-tmcode-run-dev
/usr/local/np-tmcode /usr/local/np-tmcode
%post
apt update
apt -y upgrade
apt -y install libgfortran5 libgcc-s1 libhdf5-103-1 libstdc++6 libssl3 libcurl4 libsz2 zlib1g libnghttp2-14 libidn2-0 librtmp1 libssh2-1 libpsl5 libgssapi-krb5-2 libldap-2.5-0 libzstd1 libbrotli1 libaec0 libunistring2 libgmp10 libkrb5-3 libk5crypto3 libcom-err2 libkrb5support0 libsasl2-2 libp11-kit0 libtasn1-6 libkeyutils1 libffi8 python3 python-is-python3 python3-regex hdf5-tools
rm -rf /var/lib/apt/lists/*
cd /usr/local/np-tmcode
find build -name "*.o" -exec rm -v \{\} \;
find build -name "*.gcno" -exec rm -v \{\} \;
cd src
rm -rvf cluster libnptm trapping include sphere Makefile make.inc README.md
cd ..
rm -rvf containers
cd doc
rm -rvf src/cluster /src/include /src/libntpm /src/sphere /src/trapping /src/Makefile /src/make.inc
cd build/latex
rm -rvf *.tex *.out *.sty *.ind *.log *.toc *.ilg *.idx *.aux *.eps Makefile class*.pdf
cp -a /usr/bin/ld.so /usr/bin/sh /usr/bin/dash /usr/bin/mkdir /usr/bin/rm /usr/bin/cp /usr/bin/python* /usr/bin/h5* /usr/local/bin/
rm -rvf /bin/* /usr/bin/* /sbin /usr/sbin /usr/games /usr/local/games
/usr/local/bin/mkdir -p /bin
/usr/local/bin/cp -av /usr/local/bin/* /bin/
rm -rvf /usr/local/bin/* /bin/cp /bin/rm /bin/mkdir
Bootstrap: scratch
Stage: np-tmcode-run
%files from np-tmcode-run-minimal
/ /
%runscript
PATH=/bin:/usr/local/np-tmcode/src/scripts:/usr/local/np-tmcode/build/trapping:/usr/local/np-tmcode/build/cluster:/usr/local/np-tmcode/build/sphere $*
File added
...@@ -3,13 +3,13 @@ BUILDDIR=../../build/cluster ...@@ -3,13 +3,13 @@ BUILDDIR=../../build/cluster
include ../make.inc include ../make.inc
all: clu edfb np_cluster all: clu edfb_clu np_cluster
clu: clu.o clu: clu.o
$(FC) $(FCFLAGS) -o $(BUILDDIR)/clu $(BUILDDIR)/clu.o $(LDFLAGS) $(FC) $(FCFLAGS) -o $(BUILDDIR)/clu $(BUILDDIR)/clu.o $(LDFLAGS)
edfb: edfb.o edfb_clu: edfb_clu.o
$(FC) $(FCFLAGS) -o $(BUILDDIR)/edfb $(BUILDDIR)/edfb.o $(LDFLAGS) $(FC) $(FCFLAGS) -o $(BUILDDIR)/edfb_clu $(BUILDDIR)/edfb_clu.o $(LDFLAGS)
np_cluster: $(BUILDDIR)/np_cluster.o $(BUILDDIR)/Commons.o $(BUILDDIR)/Configuration.o $(BUILDDIR)/file_io.o $(BUILDDIR)/Parsers.o $(BUILDDIR)/sph_subs.o $(BUILDDIR)/clu_subs.o $(BUILDDIR)/cluster.o np_cluster: $(BUILDDIR)/np_cluster.o $(BUILDDIR)/Commons.o $(BUILDDIR)/Configuration.o $(BUILDDIR)/file_io.o $(BUILDDIR)/Parsers.o $(BUILDDIR)/sph_subs.o $(BUILDDIR)/clu_subs.o $(BUILDDIR)/cluster.o
$(CXX) $(CXXFLAGS) -o $(BUILDDIR)/np_cluster $(BUILDDIR)/np_cluster.o $(BUILDDIR)/Commons.o $(BUILDDIR)/Configuration.o $(BUILDDIR)/file_io.o $(BUILDDIR)/Parsers.o $(BUILDDIR)/sph_subs.o $(BUILDDIR)/clu_subs.o $(BUILDDIR)/cluster.o $(CXXLDFLAGS) $(CXX) $(CXXFLAGS) -o $(BUILDDIR)/np_cluster $(BUILDDIR)/np_cluster.o $(BUILDDIR)/Commons.o $(BUILDDIR)/Configuration.o $(BUILDDIR)/file_io.o $(BUILDDIR)/Parsers.o $(BUILDDIR)/sph_subs.o $(BUILDDIR)/clu_subs.o $(BUILDDIR)/cluster.o $(CXXLDFLAGS)
...@@ -42,5 +42,5 @@ clean: ...@@ -42,5 +42,5 @@ clean:
rm -f $(BUILDDIR)/*.o rm -f $(BUILDDIR)/*.o
wipe: wipe:
rm -f $(BUILDDIR)/clu $(BUILDDIR)/edfb $(BUILDDIR)/*.o rm -f $(BUILDDIR)/clu $(BUILDDIR)/edfb_clu $(BUILDDIR)/*.o
File moved
# FC defines the fortran compiler to use. If undefined, GNU Make tries to use `f77`
ifndef FC ifndef FC
override FC=gfortran override FC=gfortran
endif endif
# FCFLAGS defines the compilation options for the fortran compiler
ifndef FCFLAGS ifndef FCFLAGS
override FCFLAGS=-std=legacy -O3 override FCFLAGS=-std=legacy -O3
endif endif
# LDFLAGS defines the default linker flags
ifndef LDFLAGS ifndef LDFLAGS
override LDFLAGS= override LDFLAGS=
endif endif
# CXX defines the default C++ compiler to use. If undefined, GNU Make tries to use g++
ifndef CXX ifndef CXX
override CXX=g++ override CXX=g++
endif endif
# HDF5_INCLUDE defines the default path to the HDF5 include files to use
ifndef HDF5_INCLUDE ifndef HDF5_INCLUDE
override HDF5_INCLUDE=/usr/include/hdf5/serial override HDF5_INCLUDE=/usr/include/hdf5/serial
endif endif
# CXXFLAGS defines the default compilation options for the C++ compiler
ifndef CXXFLAGS ifndef CXXFLAGS
override CXXFLAGS=-O3 -ggdb -pg -coverage -I$(HDF5_INCLUDE) #override CXXFLAGS=-O3 -ggdb -pg -coverage -I$(HDF5_INCLUDE)
override CXXFLAGS=-O3 -I$(HDF5_INCLUDE)
endif endif
# HDF5_LIB defines the default path to the HDF5 libraries to use
# CXXLDFLAGS defines the default linker flags to use for C++ codes
ifndef CXXLDFLAGS ifndef CXXLDFLAGS
ifndef HDF5_LIB ifndef HDF5_LIB
override HDF5_LIB=/usr/lib/x86_64-linux-gnu/hdf5/serial override HDF5_LIB=/usr/lib/x86_64-linux-gnu/hdf5/serial
endif endif
override CXXLDFLAGS=-L/usr/lib64 -L$(HDF5_LIB) -lhdf5 $(LDFLAGS) override CXXLDFLAGS=-L/usr/lib64 -L$(HDF5_LIB) -lhdf5 $(LDFLAGS)
#else
#override CXXLDFLAGS=-L/usr/lib64 -L$(HDF5_LIB) -lhdf5 $(CXXLDFLAGS)
endif endif
%.o : %.f %.o : %.f
......
...@@ -9,10 +9,10 @@ include ../make.inc ...@@ -9,10 +9,10 @@ include ../make.inc
#CXXFLAGS=-O2 -ggdb -pg -coverage #CXXFLAGS=-O2 -ggdb -pg -coverage
#CXXLFLAGS=-L/usr/lib64 -lhdf5_hl -lhdf5 #CXXLFLAGS=-L/usr/lib64 -lhdf5_hl -lhdf5
all: edfb sph np_sphere all: edfb_sph sph np_sphere
edfb: edfb.o edfb_sph: edfb_sph.o
$(FC) $(FCFLAGS) -o $(BUILDDIR)/edfb $(BUILDDIR)/edfb.o $(LDFLAGS) $(FC) $(FCFLAGS) -o $(BUILDDIR)/edfb_sph $(BUILDDIR)/edfb_sph.o $(LDFLAGS)
sph: sph.o sph: sph.o
$(FC) $(FCFLAGS) -o $(BUILDDIR)/sph $(BUILDDIR)/sph.o $(LDFLAGS) $(FC) $(FCFLAGS) -o $(BUILDDIR)/sph $(BUILDDIR)/sph.o $(LDFLAGS)
...@@ -45,5 +45,5 @@ clean: ...@@ -45,5 +45,5 @@ clean:
rm -f $(BUILDDIR)/*.o rm -f $(BUILDDIR)/*.o
wipe: wipe:
rm -f $(BUILDDIR)/edfb $(BUILDDIR)/sph $(BUILDDIR)/*.o rm -f $(BUILDDIR)/edfb_sph $(BUILDDIR)/sph $(BUILDDIR)/*.o
File moved
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment