initial commit

config.sh

+#!/bin/bash
+
+# This script can be used to apply the complete configuration of this LICCiA Base System
+# to a virtual machine
+
+APP_NAME=liccia-ubuntu
+APP_REPO="https://github.com/ls2n-dev/liccia-apps.git"
+APP_DIR="/opt/liccia"
+APP_GIT="${APP_DIR}/git/${APP_NAME}"
+
+mkdir -p `dirname $APP_GIT` && chown ubuntu: `dirname $APP_DIR` && git clone ${APP_REPO} ${APP_GIT} && \
+cd ${APP_GIT}/ubuntu && {
+  source profile_liccia.sh
+  source profile_funcs.sh
+  liccia_log INFO "start configuration process"
+  source install.sh
+}

docker-reconfigure.sh

+#!/bin/bash
+#
+# called by ansible process - system-update.yaml
+#
+# Add Docker’s official GPG key
+#curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+#echo \
+#  "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
+#  $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+usermod -aG docker ubuntu
+mkdir -p /etc/systemd/system/docker.service.d
+cat<<EOF | tee /etc/systemd/system/docker.service.d/proxy.conf
+[Service]
+Environment="HTTP_PROXY=http://172.20.12.74:3128"
+Environment="HTTPS_PROXY=http://172.20.12.74:3128"
+Environment="NO_PROXY=localhost,127.0.0.1,169.254.169.254,172.20.0.0/16,::1"
+EOF
+#
+# Configure docker to use specific local data root for images
+# append this line below in the daemon.json file before the value "bip"
+# be sure the ',' is present and the end of the line
+#   "data-root": "/specify/the/absolute/path/dir",
+#
+cat<<EOFD | sudo tee /etc/docker/daemon.json
+{
+   "bip": "192.168.250.1/24",
+   "mtu": 1500,
+   "data-root": "/opt/data/docker",
+   "runtimes": {
+        "nvidia": {
+            "path": "/usr/bin/nvidia-container-runtime",
+            "runtimeArgs": []
+        }
+    }
+}
+EOFD
+systemctl daemon-reload 
+systemctl restart docker.service
+pkill -SIGHUP dockerd

install-cuda.sh

+#!/bin/bash
+#
+
+wget https://developer.download.nvidia.com/compute/cuda/11.5.0/local_installers/cuda_11.5.0_495.29.05_linux.run
+sudo sh cuda_11.5.0_495.29.05_linux.run

install.sh

+#!/bin/bash
+# Inspired from https://gitlab.in2p3.fr/ifb-biosphere/apps/ubuntu-ifb
+# Hacked for LICCiA Environment
+
+# This script is executed on the virtual machine during the Installation phase (need to be ran as root!).
+# It is used to record a predefined VM-image of the appliance.
+# Otherwise executed first during a cloud deployement in OS-BIRD Infra
+
+# Initialization
+# source profile_liccia.sh
+# source profile_funcs.sh
+
+# Install Ansible
+# liccia_log "ansible installation in progress.."
+export DEBIAN_FRONTEND=noninteractive
+APT_OPT=""
+DIST_CODENAME=$(lsb_release -cs)
+if [ $(lsb_release -is) = "Debian" ]; then
+    apt-get install -y software-properties-common dirmngr
+	if [[ $DIST_CODENAME == "stretch" || $DIST_CODENAME == "buster" ]]; then
+        apt-add-repository "deb http://deb.debian.org/debian ${DIST_CODENAME}-backports main";
+    fi
+    APT_OPT="-t ${DIST_CODENAME}-backports --allow-unauthenticated"
+    apt-get -y --allow-unauthenticated dist-upgrade
+elif [ $DIST_CODENAME != "focal" ]; then
+    apt-add-repository -y ppa:ansible/ansible
+fi
+apt-get update
+apt-get install $APT_OPT -y ansible
+# liccia_log "ansible installation done!"
+
+# Run app playbook
+# liccia_log "updating system with ansible.."
+# liccia_log CMD "ansible-playbook -c local -i 127.0.0.1, -b -e 'ansible_python_interpreter=/usr/bin/python3' system-update.yaml"
+ansible-playbook -c local -i 127.0.0.1, -b -e 'ansible_python_interpreter=/usr/bin/python3' system-update.yaml

miniconda-install.sh

+#!/bin/bash
+# Inspired from https://gitlab.in2p3.fr/ifb-biosphere/apps/ubuntu-ifb
+# Hacked for LICCiA Environment
+#
+
+CONDA_SHELL=Miniconda3-latest-Linux-x86_64.sh
+CONDA_DIR=/var/lib/miniconda3
+#LOG_DEPLOYMENT=/var/log/miniconda-deploy.log
+
+#liccia_log "Miniconda3 installation in progress.."
+
+# Install miniconda:latest
+# 191025: Add a time-out limit on miniconda installation
+curl -LJO --connect-timeout 10 https://repo.continuum.io/miniconda/$CONDA_SHELL
+if [ ! -e $CONDA_SHELL ]; then
+   #liccia_log ERROR "Miniconda3 installation timed out"
+   exit
+fi
+timeout 1m bash $CONDA_SHELL -b -u -p $CONDA_DIR
+if [ $? -ne 0 ]; then
+   #liccia_log ERROR  "Miniconda3 installation timed out"
+   exit
+fi
+rm $CONDA_SHELL
+
+#liccia_log "Miniconda3 configuration environment.."
+
+# Configure conda environment
+ln -s $CONDA_DIR/etc/profile.d/conda.sh /etc/profile.d/conda.sh
+source /etc/profile.d/conda.sh
+
+chgrp -R ubuntu $CONDA_DIR
+chmod -R g+w $CONDA_DIR
+
+## Add conda channels
+# conda config --add channels bioconda --system
+conda config --add channels R --system
+conda config --add channels conda-forge --system
+
+## Install Mamba (and update to the very last version of conda)
+conda install -n base -c conda-forge mamba
+
+#liccia_log "Miniconda3 installation done!"

nvidia-configure.sh

+#!/bin/bash
+#
+distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \
+   && curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - \
+   && curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+sudo apt update
+cat<<EOFF | sudo tee -a /etc/profile.d/ls2n.sh
+export NVIDIA_CUDA_VERSION=$(/usr/bin/nvidia-smi | head -3  | grep "CUDA Version" | tr -d '|' | awk -F': ' '{print $3}')
+export NVIDIA_DRIVER_VERSION=$(/usr/bin/nvidia-smi | head -3  | grep "NVIDIA-SMI" | tr -d '|' | awk '{print $2}')
+EOFF

nvidia-install.sh

+#!/bin/bash
+
+apt-get install linux-headers-$(uname -r)
+apt-get purge nvidia*
+
+echo "++ update grub config"
+sed -i.bak-`date +%s` 's/^\(GRUB_CMDLINE_LINUX=\"\)\(.*\)\"/\1\2 nouveau.modeset=0\"/g' /etc/default/grub
+
+# reboot
+
+DRIVER_VERSION=418.197.02
+echo "++ downloading nvidia-driver version $DRIVER_VERSION"
+if [[ `sudo find /opt/data -maxdepth 2 -iname 'NVIDIA-Linux*' | wc -l` -eq 0 ]]; then 
+   BASE_URL=https://us.download.nvidia.com/tesla
+   cd /opt/data/ && sudo curl -fSsl -O $BASE_URL/$DRIVER_VERSION/NVIDIA-Linux-x86_64-$DRIVER_VERSION.run
+fi
+[ -f /opt/data/NVIDIA-Linux-x86_64-$DRIVER_VERSION.run ]] && {
+   echo "+++ installing the driver..."
+   sh /opt/data/NVIDIA-Linux-x86_64-$DRIVER_VERSION.run -s -a
+}

profile_funcs.sh

+function liccia_log(){
+  TAG='INFO'
+  case "$1" in
+    INFO | CMD)  TAG=$1; shift ;;
+    ERROR) TAG="*ERROR*"; shift ;;
+  esac
+  echo "[`date +%Y-%m-%dT%T%z`] $TAG $*" | tee -a $LICCIA_LOGFILE
+}
+export -f liccia_log

profile_liccia.sh

+# LICCiA User Environment
+export LICCIA_MAIN=/opt/liccia
+export LICCIA_BIN=$LICCIA_MAIN/bin
+export LICCIA_DATADIR=$LICCIA_MAIN/data
+export LICCIA_LOGFILE=$LICCIA_MAIN/liccia.log
+export PATH=$PATH:$LICCIA_BIN

system-update.yaml

+- hosts: all
+  tasks:
+
+  - name: Install requirements
+    apt:
+      name: 
+        - autofs
+        - fail2ban
+        - python3-pip
+        - nfs-common
+        - jq
+        - python3 
+        - python3-dev 
+        - git 
+        - curl
+        - wget
+        - python3-venv
+        - lsb-release
+      state: latest
+      update_cache: yes
+
+  - name: Remove old Docker version
+    apt:
+      name:
+        - docker
+        - docker-engine
+        - docker.io
+        - containerd
+        - runc
+      state: absent
+
+  - name: Install Docker deps
+    apt:
+      name:
+        - apt-transport-https
+        - ca-certificates
+        - gnupg
+        - gnupg-agent
+        - software-properties-common
+      state: latest
+      update_cache: yes
+
+  - name: Add Docker’s official GPG key
+    apt_key:
+      url: https://download.docker.com/linux/{{ ansible_distribution | lower }}/gpg
+      state: present
+
+  - name: Add Docker's repository into sources list
+    apt_repository:
+      repo: deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} stable
+      state: present
+
+  - name: Install Docker engine community
+    apt:
+      name:
+        - docker-ce
+        - docker-ce-cli
+        - containerd.io
+      state: latest
+
+  - name: Configure docker
+    script: docker-reconfigure.sh
+
+  - name: Run userdata script
+    script: userdata-install.sh
+
+- name: Enable services
+    systemd:
+      name: "{{ item }}" 
+      enabled: yes
+      state: started
+    loop:
+      - docker
+      - fail2ban
+
+  - name: Update pip
+    pip:
+      name: 
+        - pip
+      executable: pip3
+      state: latest
+
+  - name: Install docker-compose
+    pip:
+      name: 
+        - docker-compose
+      executable: pip3
+      state: latest
+
+  - name: Copy LICCiA profile
+    copy:
+      src: profile_liccia.sh
+      dest: /etc/profile.d/liccia_env.sh
+      owner: root
+      mode: '0644'
+
+  - name: Create LICCiA root dir and bin
+    shell: . /etc/profile.d/liccia_env.sh && mkdir -p $LICCIA_BIN
+
+  - name: Create LICCiA data dir
+    shell: . /etc/profile.d/liccia_env.sh && mkdir -p $LICCIA_DATADIR

userdata-bare.sh

+#!/bin/bash -lxe
+# Python 3.6
+#
+export LC_ALL=C
+export DISTRIB_CODENAME=$(lsb_release -cs)
+cat<<EOF | tee /home/ubuntu/.liccia-output.log
+-- start-configuration: `date`
+EOF
+
+# Clean up docker installation
+apt-get purge -y docker-engine docker docker.io docker-ce docker-ce-cli
+apt-get autoremove -y --purge docker-engine docker docker.io docker-ce  
+rm -rf /etc/docker /var/lib/docker /etc/apt/sources.list.d/docker.list 2>/dev/null
+rm -f /etc/apparmor.d/docker
+groupdel docker
+rm -rf /var/run/docker.sock
+
+# Set up the repository
+dpkg --configure -a
+apt-get update && \
+apt-get install -y python3 python3-dev python3-pip git curl wget \
+    apt-transport-https python3-pip python3-venv \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+
+python3 -m pip install --upgrade pip 
+cat<<EOF | tee -a /home/ubuntu/.liccia-output.log
+-- end-configuration: `date`
+EOF
+chown ubuntu: /home/ubuntu/.liccia-output.log

userdata-docker.sh

+#!/bin/bash -lxe
+# source: https://docs.docker.com/engine/install/ubuntu
+# Python 3.6
+# Docker default 5:19.03.15~3-0
+# input DOCKER_VERSION
+#
+export LC_ALL=C
+export DISTRIB_CODENAME=$(lsb_release -cs)
+
+# To find a specific version, run "apt-cache madison docker-ce"
+#
+export DEFAULT_DOCKER_VERSION="5:19.03.15~3-0"
+export DOCKER_VERSION="${DOCKER_VERSION:-${DEFAULT_DOCKER_VERSION}}~ubuntu-${DISTRIB_CODENAME}"
+cat<<EOF | tee /home/ubuntu/.liccia-output.log
+-- start-configuration: `date`
+EOF
+
+# Set up the repository
+dpkg --configure -a
+apt-get update && \
+apt-get install -y python3 python3-dev python3-pip git curl wget \
+    apt-transport-https python3-pip python3-venv \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+
+# Add Docker’s official GPG key
+echo \
+  "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+# Install Docker Engine
+apt-get update && \
+apt-get install -y docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io
+
+usermod -aG docker ubuntu
+mkdir -p /etc/systemd/system/docker.service.d
+cat<<EOF | tee /etc/systemd/system/docker.service.d/proxy.conf
+[Service]
+Environment="HTTP_PROXY=http://172.20.12.74:3128"
+Environment="HTTPS_PROXY=http://172.20.12.74:3128"
+Environment="NO_PROXY=localhost,127.0.0.1,169.254.169.254,172.20.0.0/16,::1"
+EOF
+cat<<EOF1 | tee /etc/docker/daemon.json
+{
+   "bip": "192.168.250.1/24",
+   "mtu": 1500
+}
+EOF1
+systemctl daemon-reload 
+systemctl restart docker.service
+pkill -SIGHUP dockerd
+python3 -m pip install --upgrade pip 
+python3 -m pip install docker-compose 
+cat<<EOF | tee -a /home/ubuntu/.liccia-output.log
+-- end-configuration: `date`
+EOF
+chown ubuntu: /home/ubuntu/.liccia-output.log

userdata-install.sh

+#!/bin/bash

userdata-mountpoint.sh

+#!/bin/bash
+#
+export DEV=/dev/sdb
+export MOUNTPOINT=/opt/data
+[ ! -d $MOUNTPOINT ] && sudo mkdir -p $MOUNTPOINT 2>/dev/null
+# add in fstab disk mount point
+if [[ `dmesg | grep -Ei '[sv]db' | grep 'Attached'` ]]; then
+   if [ -z "`grep $DEV /etc/fstab 2>/dev/null`" ]; then
+      sudo sh -c "echo '${DEV}1 $MOUNTPOINT ext4 defaults 0 0' >> /etc/fstab"
+      sudo mount -a
+   fi
+fi