Merge pull request #348 from advancedtelematic/ci/dockerize

Ci/dockerize

5 files changed, 108 insertions, 29 deletions

diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake
new file mode 100644
index 0000000..4dfafec
--- /dev/null
+++ b/scripts/ci/Dockerfile.bitbake
@@ -0,0 +1,36 @@
+FROM debian:stable
+LABEL Description="Image for bitbaking"
+
+RUN sed -i 's#deb http://deb.debian.org/debian stable main#deb http://deb.debian.org/debian stable main contrib#g' /etc/apt/sources.list
+RUN sed -i 's#deb http://deb.debian.org/debian stable-updates main#deb http://deb.debian.org/debian stable-updates main contrib#g' /etc/apt/sources.list
+RUN apt-get update -q && apt-get install -qy \
+     build-essential \
+     bzip2 \
+     chrpath \
+     cpio \
+     default-jre \
+     diffstat \
+     gawk \
+     gcc-multilib \
+     git-core \
+     iputils-ping \
+     iproute \
+     libpython-dev \
+     libsdl1.2-dev \
+     locales \
+     procps \
+     python \
+     python3 \
+     python3-pexpect \
+     qemu \
+     socat \
+     texinfo \
+     unzip \
+     wget \
+     xterm \
+     xz-utils
+
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
+ENV LC_ALL="en_US.UTF-8"
+ENV LANG="en_US.UTF-8"
+ENV LANGUAGE="en_US.UTF-8"
diff --git a/scripts/ci/Jenkinsfile b/scripts/ci/Jenkinsfile.bleeding
index 83bd49c..6d0f1e7 100644
--- a/scripts/ci/Jenkinsfile
+++ b/scripts/ci/Jenkinsfile.bleeding
@@ -1,24 +1,31 @@
+// This CI setup checks out aktualizr, meta-updater and updater-repo and builds
+// master branches whenever a change is pushed to any of these
+
 pipeline {
   agent none
   environment {
-    TEST_LOCAL_CONF_APPEND = 'scripts/ci/local.conf.append'
+    TEST_AKTUALIZR_REMOTE = 'aktualizr'
     TEST_AKTUALIZR_DIR = 'aktualizr'
-    TEST_AKTUALIZR_BRANCH = 'origin/master'
+    TEST_AKTUALIZR_BRANCH = 'master'
+    TEST_BITBAKE_COMMON_DIR = "/opt/jenkins/bitbake-common"
   }
   stages {
     stage('checkout') {
-      agent {
-        label 'bitbake'
-      }
+      agent any
       steps {
-        dir('aktualizr') {
-          checkout([$class: 'GitSCM',
-              userRemoteConfigs: [[url: 'https://github.com/advancedtelematic/aktualizr']],
-              branches: [[name: '*/master']],
-              changelog: true,
-              poll: true,
-          ])
-        }
+        checkout([$class: 'GitSCM',
+            userRemoteConfigs: [
+              [url: 'https://github.com/advancedtelematic/aktualizr', name: 'aktualizr']
+            ],
+            branches: [[name: 'refs/heads/master']],
+            extensions: [
+              [$class: 'DisableRemotePoll'],
+              [$class: 'PruneStaleBranch'],
+              [$class: 'RelativeTargetDirectory',
+                relativeTargetDir: 'aktualizr'
+              ]
+            ],
+        ])
 
         checkout([$class: 'RepoScm',
             manifestRepositoryUrl: 'https://github.com/advancedtelematic/updater-repo',
@@ -38,6 +45,9 @@ pipeline {
             showAllChanges: false,
         ])
 
+        // ignore bitbake build directories in docker
+        sh 'echo \'build*\' > .dockerignore'
+
         // override meta-updater commit with currently tested branch
         sh '''
            META_UPDATER_COMMIT=$(git rev-parse HEAD)
@@ -48,7 +58,10 @@ pipeline {
     }
     stage('build-core-image-minimal') {
       agent {
-        label 'bitbake'
+        dockerfile {
+          filename 'scripts/ci/Dockerfile.bitbake'
+          args '-v /opt/jenkins/bitbake-common:/opt/jenkins/bitbake-common'
+        }
       }
       steps {
         sh 'scripts/ci/configure.sh'
diff --git a/scripts/ci/README.adoc b/scripts/ci/README.adoc
new file mode 100644
index 0000000..222982b
--- /dev/null
+++ b/scripts/ci/README.adoc
@@ -0,0 +1,14 @@
+= Jenkins setup for running meta-updater CI
+
+As bitbake is quite resource-hungry, there are some special steps that are
+needed to run Jenkins CI tasks:
+
+- docker should be installed and the `jenkins` unix user should belong to
+  the `docker` group
+- `/opt/jenkins` should exist and have `jenkins:jenkins` permissions, it
+  will be mapped as a volume on the same location in the docker build
+  container
+
+Note that for nodes running Jenkins slaves as a docker container, the
+`/opt/jenkins` directory must exist on the host system as well, with
+permissions matching the user and groupd ids in Jenkins' docker
diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh
index 36ed059..1e87a7b 100755
--- a/scripts/ci/configure.sh
+++ b/scripts/ci/configure.sh
@@ -8,11 +8,11 @@ TEST_BUILD_DIR=${TEST_BUILD_DIR:-build}
 TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo}
 
 TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.}
-TEST_LOCAL_CONF_APPEND=${TEST_LOCAL_CONF_APPEND:-}
 TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master}
-TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="${TEST_AKTUALIZR_DIR}/.git" git rev-parse "${TEST_AKTUALIZR_BRANCH}")}
+TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")}
+TEST_BITBAKE_COMMON_DIR=${TEST_BITBAKE_COMMON_DIR:-}
 
-# remove existing local.conf, keep
+# move existing conf directory to backup, before generating a new one
 rm -rf "${TEST_BUILD_DIR}/conf.old" || true
 mv "${TEST_BUILD_DIR}/conf" "${TEST_BUILD_DIR}/conf.old" || true
 
@@ -23,11 +23,32 @@ echo ">> Running envsetup.sh"
 . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}"
 )
 
-if [[ -n $TEST_LOCAL_CONF_APPEND ]]; then
-    echo ">> Appending to local.conf"
-    REMOTE_AKTUALIZR_BRANCH=$(sed 's#^[^/]*/##g' <<< "$TEST_AKTUALIZR_BRANCH")
-    cat "$TEST_LOCAL_CONF_APPEND" | \
-        sed "s/\$<rev-sha1>/$TEST_AKTUALIZR_REV/g" | \
-        sed "s/\$<rev-branch>/$REMOTE_AKTUALIZR_BRANCH/g" \
-        >> "${TEST_BUILD_DIR}/conf/local.conf"
+set +x
+
+echo ">> Set common bitbake config options"
+cat << EOF > "${TEST_BUILD_DIR}/conf/site.conf"
+SANITY_TESTED_DISTROS = ""
+SSTATE_MIRRORS ?= "file://.* https://bitbake-cache.atsgarage.com/PATH;downloadfilename=PATH"
+IMAGE_FEATURES += "ssh-server-openssh"
+
+EOF
+
+echo ">> Set aktualizr branch in bitbake's config"
+cat << EOF >> "${TEST_BUILD_DIR}/conf/site.conf"
+SRCREV_pn-aktualizr = "$TEST_AKTUALIZR_REV"
+SRCREV_pn-aktualizr-native = "\${SRCREV_pn-aktualizr}"
+BRANCH_pn-aktualizr = "$TEST_AKTUALIZR_BRANCH"
+BRANCH_pn-aktualizr-native = "\${BRANCH_pn-aktualizr}"
+
+EOF
+
+if [[ -n $TEST_BITBAKE_COMMON_DIR ]]; then
+    echo ">> Set caching"
+    SSTATE_DIR="$TEST_BITBAKE_COMMON_DIR/sstate-cache"
+    DL_DIR="$TEST_BITBAKE_COMMON_DIR/downloads"
+    mkdir -p "$SSTATE_DIR" "$DL_DIR"
+    cat << EOF >> "${TEST_BUILD_DIR}/conf/site.conf"
+SSTATE_DIR = "$SSTATE_DIR"
+DL_DIR = "$DL_DIR"
+EOF
 fi
diff --git a/scripts/ci/local.conf.append b/scripts/ci/local.conf.append
deleted file mode 100644
index 350e466..0000000
--- a/scripts/ci/local.conf.append
+++ /dev/null
@@ -1,5 +0,0 @@
-SANITY_TESTED_DISTROS = ""
-SRCREV_pn-aktualizr = "$<rev-sha1>"
-SRCREV_pn-aktualizr-native = "${SRCREV_pn-aktualizr}"
-BRANCH_pn-aktualizr = "$<rev-branch>"
-BRANCH_pn-aktualizr-native = "${BRANCH_pn-aktualizr}"