Merge pull request #30 from icgc-argo/pre-issue29

junjun-zhang · web-flow · commit a27566cbd555 · 2021-04-01T10:42:05.000-04:00
added files for wfpm
diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml
@@ -0,0 +1,253 @@
+name: CI/CD
+
+on:
+  push:
+    branches:
+    - main
+    - '*\@[0-9]+.[0-9]+.[0-9]+*'  # package development branch
+
+jobs:
+  build:
+    if: ${{ !contains(github.event.head_commit.message, '[no ci]') }}
+    runs-on: ubuntu-latest
+    outputs:
+      branch: ${{ steps.get_pkg_info.outputs.branch }}
+      pkg_name: ${{ steps.get_pkg_info.outputs.pkg_name }}
+      pkg_ver: ${{ steps.get_pkg_info.outputs.pkg_ver }}
+      docker_file: ${{ steps.get_pkg_info.outputs.docker_file }}
+      repo_lowercase: ${{ steps.repo_lowercase.outputs.lowercase }}
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+
+    - name: Extract package name and version from branch name
+      id: get_pkg_info
+      shell: bash
+      run: |
+        echo "::set-output name=branch::$(echo ${GITHUB_REF#refs/heads/})"
+        PKG_NAME=$(echo ${GITHUB_REF#refs/heads/} | awk -F'@' '{print $1}')
+        PKG_VER=$(echo ${GITHUB_REF#refs/heads/} | awk -F'@' '{print $2}')
+        echo "::set-output name=pkg_name::$(echo $PKG_NAME)"
+        echo "::set-output name=pkg_ver::$(echo $PKG_VER)"
+        if [[ -f "./$PKG_NAME/Dockerfile" ]]; then
+          echo "::set-output name=docker_file::$(echo ./$PKG_NAME/Dockerfile)"
+        else
+          echo "::set-output name=docker_file::"
+        fi
+
+    - name: Login to GitHub Container Registry
+      if: ${{ steps.get_pkg_info.outputs.branch != 'main' && steps.get_pkg_info.outputs.docker_file != ''}}
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.CR_PAT }}
+
+    - id: repo_lowercase
+      uses: ASzc/change-string-case-action@v1
+      with:
+        string: ${{ github.repository }}
+
+    - name: Build image
+      if: ${{ steps.get_pkg_info.outputs.branch != 'main' && steps.get_pkg_info.outputs.docker_file != ''}}
+      uses: docker/build-push-action@v2
+      with:
+        context: "./${{steps.get_pkg_info.outputs.pkg_name}}"
+        file: "${{steps.get_pkg_info.outputs.docker_file}}"
+        platforms: linux/amd64
+        load: true
+        tags: |
+          ghcr.io/${{ steps.repo_lowercase.outputs.lowercase }}.${{ steps.get_pkg_info.outputs.pkg_name }}:${{ steps.get_pkg_info.outputs.pkg_ver }}
+
+    - name: Push image to ghcr.io
+      id: push_to_ghcr
+      if: ${{ steps.get_pkg_info.outputs.branch != 'main' && steps.get_pkg_info.outputs.docker_file != ''}}
+      shell: bash
+      run: |
+        docker push ghcr.io/${{ steps.repo_lowercase.outputs.lowercase }}.${{ steps.get_pkg_info.outputs.pkg_name }}:${{ steps.get_pkg_info.outputs.pkg_ver }} | tee ./ghcr.io.stdout
+        IMAGE_SHA=$(cat ./ghcr.io.stdout | tr ' ' '\n' | grep 'sha256:' | awk -F':' '{print $2}')
+        echo "::set-output name=image_sha256::$(echo ${IMAGE_SHA})"
+        echo "::set-output name=created_at::$(date -u +%FT%TZ)"
+
+
+  test-and-release:
+    runs-on: ubuntu-latest
+    needs: [build]
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install wfpm
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi
+        if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+    - name: Install Nextflow
+      run: |
+        wget --tries=10 -qO- https://get.nextflow.io | bash
+        sudo chmod 755 nextflow
+        sudo mv nextflow /usr/local/bin/
+
+    - name: "Login to GitHub Container Registry"  # normally shouldn't need to login, but new image when just created is private
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.CR_PAT }}
+
+    - name: Run tests for all packages
+      if: ${{ needs.build.outputs.branch == 'main' }}
+      run: |
+        wfpm test
+
+    - name: Run tests for the current package only
+      if: ${{ needs.build.outputs.branch != 'main' }}
+      run: |
+        cd ./${{needs.build.outputs.pkg_name}}
+        wfpm test
+
+    - name: Check whether to proceed with release
+      id: to_release
+      run: |
+        if [[ \
+          "${{ github.event.head_commit.message }}" = "Merge"[[:space:]]"pull"[[:space:]]"request"* && \
+          "${{ github.event.head_commit.message }}" = *"@"* && \
+          "${{ github.event.head_commit.message }}" = *"[release]"* && \
+          "${{ needs.build.outputs.branch }}" = 'main'
+        ]]; then
+          echo "::set-output name=release::$(echo Y)"
+        else
+          echo "::set-output name=release::$(echo N)"
+        fi
+
+    - name: Extract package name and version from commit message
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      id: get_pkg_info
+      shell: bash
+      run: |
+        MERGED_BR=$(echo -e '${{ github.event.head_commit.message }}' | \
+          { grep '^Merge pull request'||:; } | tr ' ' '\n' |{ grep '@'||:; } | awk -F'/' '{print $2}')
+        PKG_NAME=$(echo $MERGED_BR | awk -F'@' '{print $1}')
+        PKG_VER=$(echo $MERGED_BR | awk -F'@' '{print $2}')
+        echo "::set-output name=pkg_name::$(echo ${PKG_NAME})"
+        echo "::set-output name=pkg_ver::$(echo ${PKG_VER})"
+        if [[ -f "./$PKG_NAME/Dockerfile" ]]; then
+          echo "::set-output name=docker_file::$(echo ./$PKG_NAME/Dockerfile)"
+          echo "::set-output name=docker_image::$(echo ghcr.io/${{ needs.build.outputs.repo_lowercase }}.$PKG_NAME:$PKG_VER)"
+        else
+          echo "::set-output name=docker_file::"
+          echo "::set-output name=docker_image::"
+        fi
+
+    - name: Prepare package release tarball
+      id: prep_assets
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      shell: bash
+      run: |
+        ./scripts/cleanup_temp_files.sh  # just in case
+        PKG_TAR=${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}.tar.gz
+        pushd ${{ steps.get_pkg_info.outputs.pkg_name }}
+        tar --exclude=wfpr_modules --dereference -czvf ../$PKG_TAR .
+        popd
+        echo "::set-output name=pkg_tar::$(echo ${PKG_TAR})"
+        echo "::set-output name=pkg_tar_sha::$(sha256sum ${PKG_TAR} |awk '{print $1}')"
+        echo "::set-output name=pkg_tar_size::$(ls -l ${PKG_TAR} |awk '{print $5}')"
+        echo "::set-output name=created_at::$(date -u +%FT%TZ)"
+
+    - name: Create release
+      id: create_release
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}
+        release_name: ${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}
+        body: |
+          * Release `${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}` (${{ github.sha }})
+          * Package `${{ steps.prep_assets.outputs.pkg_tar }}` (sha256: `${{ steps.prep_assets.outputs.pkg_tar_sha }}`)
+        draft: false
+        prerelease: false
+
+    - name: Prepare in pkg-release.json
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      shell: bash
+      run: |
+        WFPM_VER=$(wfpm -v | cut -d ' ' -f 2)
+        if [[ '${{ steps.get_pkg_info.outputs.docker_file }}' != '' ]]; then
+          docker pull ${{ steps.get_pkg_info.outputs.docker_image }} | tee image.info
+          IMAGE_SHA=$(cat ./image.info | tr ' ' '\n' | grep 'sha256:' | awk -F':' '{print $2}')
+        else
+          IMAGE_SHA=$(echo)
+        fi
+
+        # temporary solution
+        TAG="${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}"
+        ./scripts/prepare_package_release_json.py -p ${{ steps.get_pkg_info.outputs.pkg_name }}/pkg.json -d \
+          "
+          {
+            \"_release\": {
+              \"created\": \"${{ steps.prep_assets.outputs.created_at }}\",
+              \"hash\": {
+                \"checksum_type\": \"sha1\",
+                \"checksume\": \"${{ github.sha }}\"
+              },
+              \"tag\": \"$TAG\",
+              \"assets\": [
+                {
+                  \"filename\": \"$TAG.tar.gz\",
+                  \"download_url\": \"https://github.com/${{ github.repository }}/releases/download/$TAG/$TAG.tar.gz\",
+                  \"checksum\": \"${{ steps.prep_assets.outputs.pkg_tar_sha }}\",
+                  \"checksum_type\": \"sha256\",
+                  \"size\": ${{ steps.prep_assets.outputs.pkg_tar_size }}
+                },
+                {
+                  \"_NOTE_\": \"this file\",
+                  \"filename\": \"pkg-release.json\",
+                  \"download_url\": \"https://github.com/${{ github.repository }}/releases/download/$TAG/pkg-release.json\",
+                  \"checksum\": null,
+                  \"checksum_type\": null,
+                  \"size\": null
+                }
+              ]
+            },
+            \"_image_digest\": {
+              \"checksum\": \"$IMAGE_SHA\",
+              \"checksum_type\": \"sha256\"
+            },
+            \"_wfpm_ver\": \"$WFPM_VER\"
+          }
+          " > pkg-release.json
+
+    - name: Upload package release tarball
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./${{ steps.prep_assets.outputs.pkg_tar }}
+        asset_name: ${{ steps.prep_assets.outputs.pkg_tar }}
+        asset_content_type: application/zip
+
+    - name: Upload package release json
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./pkg-release.json
+        asset_name: pkg-release.json
+        asset_content_type: application/json
diff --git a/.gitignore b/.gitignore
@@ -10,4 +10,9 @@ reference
 out
 reports
 remote_reports
-params.json
+params.json
+.log
+.python-version
+.nextflow*
+outdir
+wfpr_modules/github.com/*/*/*/tests
diff --git a/.wfpm b/.wfpm
@@ -0,0 +1,5 @@
+project_name: nextflow-data-processing-utility-tools
+license: Apache License 2.0
+repo_type: git
+repo_server: github.com
+repo_account: icgc-argo
diff --git a/LICENSE-short b/LICENSE-short
@@ -0,0 +1,13 @@
+Copyright (c) 2020-2021, Ontario Institute for Cancer Research
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/nextflow.config b/nextflow.config
@@ -1,3 +1,10 @@
+manifest {
+    homePage = 'https://github.com/icgc-argo/nextflow-data-processing-utility-tools'
+    description = 'SONG/SCORE download/upload packages'
+    nextflowVersion = '>=20.10'
+}
+
 docker {
     enabled = true
+    runOptions = '-u \$(id -u):\$(id -g)'
 }
diff --git a/scripts/cleanup_temp_files.sh b/scripts/cleanup_temp_files.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+for f in `find . -type d |egrep '(/work$)|(/.nextflow$)|(/outdir$)|(.nextflow.log)'`; do
+    echo remove $f;
+    rm -fr $f;
+done
+
+for f in `find . -type f |egrep '\.nextflow\.log'`; do
+    echo remove $f;
+    rm $f;
+done
+
diff --git a/scripts/prepare_package_release_json.py b/scripts/prepare_package_release_json.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+# this is a temporary solution
+
+import argparse
+import json
+
+
+def update_image_digest(package_meta, release_meta_str):
+    with open(package_meta, 'r') as f:
+        package_meta = json.load(f)
+
+    release_meta = json.loads(release_meta_str)
+
+    if 'container' in package_meta:
+        package_meta['container']['_image_digest'] = release_meta['_image_digest']
+    package_meta['_release'] = release_meta['_release']
+    package_meta['_wfpm_ver'] = release_meta.get('_wfpm_ver')
+
+    return package_meta
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Update pkg.json to generate pkg-release.json ')
+    parser.add_argument('-p', dest='package_meta', type=str, required=True)
+    parser.add_argument('-d', dest='release_meta_str', type=str, required=True)
+    args = parser.parse_args()
+
+    updated_meta = update_image_digest(
+        args.package_meta,
+        args.release_meta_str
+    )
+
+    print(json.dumps(updated_meta, indent=4, sort_keys=True))
diff --git a/wfpr_modules/README.md b/wfpr_modules/README.md
@@ -0,0 +1,10 @@
+# Workflow Package Installation Directory
+
+This directory contains all dependent workflow packages that are to be installed by the package
+management tool: [WFPM](https://github.com/icgc-argo/wfpm) (WorkFlow Package Manager). Please
+do NOT modify contents under this directory manually.
+
+Ideally the packages installed in this directory do not need to be checked into git repo, similar
+to NPM's `node_modules` directory. Instead, all dependent packages/modules are to be installed
+at deployment time. But let's keep the modules checked into git repo before deployment time
+installation is supported.
