update dockerfiles for use of both CI - GHA and Travis

and do minor cleanup in docker scripts.

Author: lukaszstolarczuk

.github/workflows/gha.yml

@@ -11,6 +11,7 @@ name: CPP
 on: [push, pull_request]
 
 env:
+    REPO:           libpmemobj-cpp
     GITHUB_REPO:    pmem/libpmemobj-cpp
     DOCKERHUB_REPO: pmem/libpmemobj-cpp
 
@@ -19,6 +20,8 @@ jobs:
     name: Linux
     runs-on: ubuntu-latest
     env:
+      DOCKERHUB_USER:          ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_PASSWORD:      ${{ secrets.DOCKERHUB_PASSWORD }}
       HOST_WORKDIR:   /home/runner/work/libpmemobj-cpp/libpmemobj-cpp
       WORKDIR:        utils/docker
     strategy:
@@ -39,9 +42,6 @@ jobs:
          with:
             fetch-depth: 50
 
-       - name: Change ownership of the repo
-         run: sudo chown -R 1000.1000 $HOST_WORKDIR
-
        - name: Pull or rebuild the image
          run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh
 

utils/docker/build.sh

@@ -32,44 +32,47 @@
 
 #
 # build-local.sh - runs a Docker container from a Docker image with environment
-#                  prepared for running libpmemobj-cpp tests and run those tests.
+#                  prepared for running libpmemobj-cpp builds and tests.
 #
 #
 # Notes:
 # - run this script from its location or set the variable 'HOST_WORKDIR' to
 #   where the root of this project is on the host machine,
 # - set variables 'OS' and 'OS_VER' properly to a system you want to build this
 #	repo on (for proper values take a look on the list of Dockerfiles at the
-#   utils/docker/images directory), e.g. OS=ubuntu, OS_VER=19.04.
+#   utils/docker/images directory), e.g. OS=ubuntu, OS_VER=19.10.
 #
 
 set -e
+
+source $(dirname $0)/set-ci-vars.sh
+source $(dirname $0)/set-vars.sh
 source $(dirname $0)/valid-branches.sh
 
-# Environment variables that can be customized (default values are after dash):
-export KEEP_CONTAINER=${KEEP_CONTAINER:-0}
+if [[ "$CI_EVENT_TYPE" != "cron" && "$CI_BRANCH" != "coverity_scan" \
+	&& "$TYPE" == "coverity" ]]; then
+	echo "INFO: Skip Coverity scan job if build is triggered neither by " \
+		"'cron' nor by a push to 'coverity_scan' branch"
+	exit 0
+fi
 
+if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\
+	&& "$TYPE" != "coverity" ]]; then
+	echo "INFO: Skip regular jobs if build is triggered either by 'cron'" \
+		" or by a push to 'coverity_scan' branch"
+	exit 0
+fi
 
 if [[ -z "$OS" || -z "$OS_VER" ]]; then
 	echo "ERROR: The variables OS and OS_VER have to be set " \
-		"(e.g. OS=ubuntu, OS_VER=19.04)."
+		"(eg. OS=fedora, OS_VER=31)."
 	exit 1
 fi
 
 if [[ -z "$HOST_WORKDIR" ]]; then
-	HOST_WORKDIR=$(readlink -f ../..)
-fi
-
-if [[ "$TRAVIS_EVENT_TYPE" == "cron" || "$TRAVIS_BRANCH" == "coverity_scan" ]]; then
-	if [[ "$TYPE" != "coverity" ]]; then
-		echo "Skipping non-Coverity job for cron/Coverity build"
-		exit 0
-	fi
-else
-	if [[ "$TYPE" == "coverity" ]]; then
-		echo "Skipping Coverity job for non cron/Coverity build"
-		exit 0
-	fi
+	echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \
+		"the root of this project on the host machine"
+	exit 1
 fi
 
 imageName=${DOCKERHUB_REPO}:1.9-${OS}-${OS_VER}
@@ -114,20 +117,20 @@ fi
 if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi
 
 # Only run doc update on $GITHUB_REPO master or stable branch
-if [[ -z "${TRAVIS_BRANCH}" || -z "${TARGET_BRANCHES[${TRAVIS_BRANCH}]}" || "$TRAVIS_PULL_REQUEST" != "false" || "$TRAVIS_REPO_SLUG" != "${GITHUB_REPO}" ]]; then
+if [[ -z "${CI_BRANCH}" || -z "${TARGET_BRANCHES[${CI_BRANCH}]}" || "$CI_EVENT_TYPE" == "pull_request" || "$CI_REPO_SLUG" != "${GITHUB_REPO}" ]]; then
 	AUTO_DOC_UPDATE=0
 fi
 
-WORKDIR=/libpmemobj-cpp
-SCRIPTSDIR=$WORKDIR/utils/docker
-
-# check if we are running on a CI (Travis or GitHub Actions)
+# Check if we are running on a CI (Travis or GitHub Actions)
 [ -n "$GITHUB_ACTIONS" -o -n "$TRAVIS" ] && CI_RUN="YES" || CI_RUN="NO"
 
 # do not allocate a pseudo-TTY if we are running on GitHub Actions
 [ ! $GITHUB_ACTIONS ] && TTY='-t' || TTY=''
 
-echo Building ${OS}-${OS_VER}
+WORKDIR=/libpmemobj-cpp
+SCRIPTSDIR=$WORKDIR/utils/docker
+
+echo Building on ${OS}-${OS_VER}
 
 # Run a container with
 #  - environment variables set (--env)
@@ -138,21 +141,26 @@ docker run --privileged=true --name=$containerName -i $TTY \
 	${docker_opts} \
 	--env http_proxy=$http_proxy \
 	--env https_proxy=$https_proxy \
-	--env AUTO_DOC_UPDATE=$AUTO_DOC_UPDATE \
-	--env GITHUB_TOKEN=$GITHUB_TOKEN \
 	--env WORKDIR=$WORKDIR \
 	--env SCRIPTSDIR=$SCRIPTSDIR \
-	--env TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG \
-	--env TRAVIS_BRANCH=$TRAVIS_BRANCH \
-	--env TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE \
+	--env COVERAGE=$COVERAGE \
+	--env AUTO_DOC_UPDATE=$AUTO_DOC_UPDATE \
+	--env CI_RUN=$CI_RUN \
+	--env TRAVIS=$TRAVIS \
+	--env GITHUB_REPO=$GITHUB_REPO \
+	--env CI_COMMIT_RANGE=$CI_COMMIT_RANGE \
+	--env CI_COMMIT=$CI_COMMIT \
+	--env CI_REPO_SLUG=$CI_REPO_SLUG \
+	--env CI_BRANCH=$CI_BRANCH \
+	--env CI_EVENT_TYPE=$CI_EVENT_TYPE \
+	--env GITHUB_TOKEN=$GITHUB_TOKEN \
 	--env COVERITY_SCAN_TOKEN=$COVERITY_SCAN_TOKEN \
 	--env COVERITY_SCAN_NOTIFICATION_EMAIL=$COVERITY_SCAN_NOTIFICATION_EMAIL \
-	--env COVERAGE=$COVERAGE \
 	--env CHECK_CPP_STYLE=${CHECK_CPP_STYLE:-ON} \
 	--env TESTS_LONG=${TESTS_LONG:-OFF} \
 	--env TESTS_TBB=${TESTS_TBB:-ON} \
-	--env CI_RUN=$CI_RUN \
 	--env TZ='Europe/Warsaw' \
+	--shm-size=4G \
 	-v $HOST_WORKDIR:$WORKDIR \
 	-v /etc/localtime:/etc/localtime \
 	-w $SCRIPTSDIR \

utils/docker/images/README.md

@@ -5,13 +5,13 @@ development process vehicles and part of continuous integration process.
 
 Images built out of those recipes may by used with docker or podman as
 development environment.
-Only those used on travis are fully tested on a daily basis.
-In case of any problem, patches and github issues are welcome.
+Only those used on Travis/Github Actions are fully tested on a daily basis.
+In case of any problem, patches and Github issues are welcome.
 
 # How to build docker image
 
 ```sh
-docker build --build-arg https_proxy=https://proxy.com:port --build-arg http_proxy=http://proxy.com:port -t libpmemobj-cpp:debian-unstable -f ./Dockerfile.debian-unstable .
+docker build --build-arg https_proxy=http://proxy.com:port --build-arg http_proxy=http://proxy.com:port -t libpmemobj-cpp:debian-unstable -f ./Dockerfile.debian-unstable .
 ```
 
 # How to use docker image

utils/docker/images/build-image.sh

@@ -41,29 +41,32 @@
 
 set -e
 
+OS_VER=$1
+
 function usage {
 	echo "Usage:"
-	echo "    build-image.sh <DOCKERHUB_REPO> <OS-VER>"
-	echo "where <OS-VER>, for example, can be 'ubuntu-19.04', provided " \
-		"a Dockerfile named 'Dockerfile.ubuntu-19.04' exists in the " \
+	echo "    build-image.sh <OS-VER>"
+	echo "where <OS-VER>, for example, can be 'fedora-31', provided " \
+		"a Dockerfile named 'Dockerfile.fedora-31' exists in the " \
 		"current directory."
 }
 
-# Check if the first and second argument is nonempty
-if [[ -z "$1" || -z "$2" ]]; then
+# Check if the argument is not empty
+if [[ -z "$1" ]]; then
 	usage
 	exit 1
 fi
 
 # Check if the file Dockerfile.OS-VER exists
-if [[ ! -f "Dockerfile.$2" ]]; then
-	echo "ERROR: wrong argument."
+if [[ ! -f "Dockerfile.$OS_VER" ]]; then
+	echo "Error: Dockerfile.$OS_VER does not exist."
+	echo
 	usage
 	exit 1
 fi
 
 # Build a Docker image tagged with ${DOCKERHUB_REPO}:1.9-OS-VER
-docker build -t $1:1.9-$2 \
+docker build -t ${DOCKERHUB_REPO}:1.9-${OS_VER} \
 	--build-arg http_proxy=$http_proxy \
 	--build-arg https_proxy=$https_proxy \
-	-f Dockerfile.$2 .
+	-f Dockerfile.${OS_VER} .

utils/docker/images/push-image.sh

@@ -31,41 +31,42 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #
-# push-image.sh <OS-VER> - pushes the Docker image tagged with OS-VER
-#                          to the Docker Hub.
+# push-image.sh - pushes the Docker image tagged with OS-VER to the Docker Hub.
 #
-# The script utilizes $DOCKERHUB_USER and $DOCKERHUB_PASSWORD variables to log in to
-# Docker Hub. The variables can be set in the Travis project's configuration
+# The script utilizes $DOCKERHUB_USER and $DOCKERHUB_PASSWORD variables to
+# log in to the Docker Hub. The variables can be set in the CI's configuration
 # for automated builds.
 #
 
 set -e
 
-function usage {
-	echo "Usage:"
-	echo "    push-image.sh <OS-VER>"
-	echo "where <OS-VER>, for example, can be 'ubuntu-19.04', provided " \
-		"a Docker image tagged with ${DOCKERHUB_REPO}:1.9-ubuntu-19.04 exists " \
-		"locally."
-}
+if [[ -z "$OS" ]]; then
+	echo "OS environment variable is not set"
+	exit 1
+fi
+
+if [[ -z "$OS_VER" ]]; then
+	echo "OS_VER environment variable is not set"
+	exit 1
+fi
 
-# Check if the first argument is nonempty
-if [[ -z "$1" ]]; then
-	usage
+if [[ -z "${DOCKERHUB_REPO}" ]]; then
+	echo "DOCKERHUB_REPO environment variable is not set"
 	exit 1
 fi
 
+TAG="1.9-${OS}-${OS_VER}"
+
 # Check if the image tagged with ${DOCKERHUB_REPO}:1.9-OS-VER exists locally
-if [[ ! $(docker images -a | awk -v pattern="^${DOCKERHUB_REPO}:1.9-$1\$" \
+if [[ ! $(docker images -a | awk -v pattern="^${DOCKERHUB_REPO}:${TAG}\$" \
 	'$1":"$2 ~ pattern') ]]
 then
-	echo "ERROR: wrong argument."
-	usage
+	echo "ERROR: Docker image tagged ${DOCKERHUB_REPO}:${TAG} does not exists locally."
 	exit 1
 fi
 
 # Log in to the Docker Hub
 docker login -u="$DOCKERHUB_USER" -p="$DOCKERHUB_PASSWORD"
 
 # Push the image to the repository
-docker push ${DOCKERHUB_REPO}:1.9-$1
+docker push ${DOCKERHUB_REPO}:${TAG}

utils/docker/pull-or-rebuild-image.sh

@@ -49,38 +49,17 @@
 
 set -e
 
+source $(dirname $0)/set-ci-vars.sh
 source $(dirname $0)/set-vars.sh
 
-function get_commit_range_from_last_merge {
-	# get commit id of the last merge
-	LAST_MERGE=$(git log --merges --pretty=%H -1)
-	LAST_COMMIT=$(git log --pretty=%H -1)
-	if [ "$LAST_MERGE" == "$LAST_COMMIT" ]; then
-		# GitHub Actions commits its own merge in case of pull requests
-		# so the first merge commit has to be skipped.
-		LAST_MERGE=$(git log --merges --pretty=%H -2 | tail -n1)
-	fi
-	if [ "$LAST_MERGE" == "" ]; then
-		# possible in case of shallow clones
-		COMMIT_RANGE=""
-	else
-		COMMIT_RANGE="$LAST_MERGE..HEAD"
-		# make sure it works now
-		if ! git rev-list $COMMIT_RANGE >/dev/null; then
-			COMMIT_RANGE=""
-		fi
-	fi
-	echo $COMMIT_RANGE
-}
-
-if [[ "$TRAVIS_EVENT_TYPE" != "cron" && "$TRAVIS_BRANCH" != "coverity_scan" \
+if [[ "$CI_EVENT_TYPE" != "cron" && "$CI_BRANCH" != "coverity_scan" \
 	&& "$TYPE" == "coverity" ]]; then
 	echo "INFO: Skip Coverity scan job if build is triggered neither by " \
 		"'cron' nor by a push to 'coverity_scan' branch"
 	exit 0
 fi
 
-if [[ ( "$TRAVIS_EVENT_TYPE" == "cron" || "$TRAVIS_BRANCH" == "coverity_scan" )\
+if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\
 	&& "$TYPE" != "coverity" ]]; then
 	echo "INFO: Skip regular jobs if build is triggered either by 'cron'" \
 		" or by a push to 'coverity_scan' branch"
@@ -89,7 +68,7 @@ fi
 
 if [[ -z "$OS" || -z "$OS_VER" ]]; then
 	echo "ERROR: The variables OS and OS_VER have to be set properly " \
-             "(e.g. OS=ubuntu, OS_VER=19.04)."
+             "(eg. OS=fedora, OS_VER=31)."
 	exit 1
 fi
 
@@ -99,35 +78,11 @@ if [[ -z "$HOST_WORKDIR" ]]; then
 	exit 1
 fi
 
-# TRAVIS_COMMIT_RANGE is usually invalid for force pushes - fix it when used
-# with non-upstream repository
-if [ -n "$TRAVIS_COMMIT_RANGE" -a "$TRAVIS_REPO_SLUG" != "$GITHUB_REPO" ]; then
-	if ! git rev-list $TRAVIS_COMMIT_RANGE; then
-		TRAVIS_COMMIT_RANGE=$(get_commit_range_from_last_merge)
-	fi
-fi
-
-# Fix Travis commit range
-if [ -n "$TRAVIS_COMMIT_RANGE" ]; then
-	# $TRAVIS_COMMIT_RANGE contains "..." instead of ".."
-	# https://github.com/travis-ci/travis-ci/issues/4596
-	PR_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE/.../..}"
-fi
-
-# Set the commit range in case of GitHub Actions
-if [ -n "$GITHUB_ACTIONS" ]; then
-	PR_COMMIT_RANGE=$(get_commit_range_from_last_merge)
-fi
-
 # Find all the commits for the current build
-if [ -n "$PR_COMMIT_RANGE" ]; then
-	commits=$(git rev-list $PR_COMMIT_RANGE)
-elif [ -n "$TRAVIS" ]; then
-	commits=$TRAVIS_COMMIT
-elif [ -n "$GITHUB_ACTIONS" ]; then
-	commits=$GITHUB_SHA
+if [ -n "$CI_COMMIT_RANGE" ]; then
+	commits=$(git rev-list $CI_COMMIT_RANGE)
 else
-	commits=$(git log --pretty=%H -1)
+	commits=$CI_COMMIT
 fi
 
 echo "Commits in the commit range:"
@@ -152,17 +107,17 @@ for file in $files; do
 		# Rebuild Docker image for the current OS version
 		echo "Rebuilding the Docker image for the Dockerfile.$OS-$OS_VER"
 		pushd $images_dir_name
-		./build-image.sh ${DOCKERHUB_REPO} ${OS}-${OS_VER}
+		./build-image.sh ${OS}-${OS_VER}
 		popd
 
 		# Check if the image has to be pushed to Docker Hub
-		# (i.e. the build is triggered by commits to the ${GITHUB_REPO}
-		# repository's master branch, and the Travis build is not
+		# (i.e. the build is triggered by commits to the $GITHUB_REPO
+		# repository's stable-* or master branch, and the CI build is not
 		# of the "pull_request" type). In that case, create the empty
 		# file.
-		if [[ "${TRAVIS_REPO_SLUG}" == "${GITHUB_REPO}" \
-			&& ($TRAVIS_BRANCH == stable-* || $TRAVIS_BRANCH == master) \
-			&& $TRAVIS_EVENT_TYPE != "pull_request"
+		if [[ "$CI_REPO_SLUG" == "$GITHUB_REPO" \
+			&& ($CI_BRANCH == stable-* || $CI_BRANCH == master) \
+			&& $CI_EVENT_TYPE != "pull_request" \
 			&& $PUSH_IMAGE == "1" ]]
 		then
 			echo "The image will be pushed to Docker Hub"