Merge branch 'main' into feature/TED-1156

Author: CaptainOfHacks

.github/workflows/unit-tests-srv.yml

@@ -3,9 +3,14 @@ name: Tests on srv server
 on:
   workflow_dispatch:
   push:
-    branches: [ feature/* , main,  hotfix/* ]
-  pull_request:
-    branches: [ main, release/* ]
+    branches: [ feature/* , main,  hotfix/*, release/* ]
+#  pull_request:
+#    branches: [ main, release/* ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
   VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
@@ -30,11 +35,11 @@ jobs:
       - name: Get RML mapper
         run: make init-rml-mapper
 
-#      - name: Get Allure
-#        run: make install-allure
+      #      - name: Get Allure
+      #        run: make install-allure
 
-#      - name: Start staging infra
-#        run: make start-project-services
+      #      - name: Start staging infra
+      #        run: make start-project-services
 
       - name: Run tests
         run: make test-all

.github/workflows/unit-tests.yml

@@ -2,35 +2,39 @@ name: Tests
 
 on:
   push:
-    branches: [ feature/* , main, hotfix/* ]
-  pull_request:
-    branches: [ main, release/* ]
+    branches: [ feature/* , main, hotfix/*, release/* ]
+#  pull_request:
+#    branches: [ main, release/* ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
     name: pyTest
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade setuptools pip wheel
-        make install
-        make install-dev
-    - name: Make envfile
-      uses: SpicyPizza/create-envfile@v1
-      with:
-        envkey_VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
-        envkey_VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
-    - name: Run unit tests
-      run: make test
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade setuptools pip wheel
+          make install
+          make install-dev
+      - name: Make envfile
+        uses: SpicyPizza/create-envfile@v1
+        with:
+          envkey_VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
+          envkey_VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+      - name: Run unit tests
+        run: make test
 #    - name: Upload coverage to Codecov
 #      uses: codecov/codecov-action@v1
 #      with:

Makefile

@@ -24,12 +24,12 @@ CAROOT = $(shell pwd)/infra/traefik/certs
 install:
 	@ echo -e "$(BUILD_PRINT)Installing the requirements$(END_BUILD_PRINT)"
 	@ pip install --upgrade pip
-	@ pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.4.3/constraints-no-providers-3.8.txt"
+	@ pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt"
 
 install-dev:
 	@ echo -e "$(BUILD_PRINT)Installing the dev requirements$(END_BUILD_PRINT)"
 	@ pip install --upgrade pip
-	@ pip install --no-cache-dir -r requirements.dev.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.4.3/constraints-no-providers-3.8.txt"
+	@ pip install --no-cache-dir -r requirements.dev.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt"
 
 test: test-unit
 
@@ -318,9 +318,6 @@ rml-mapper-path-add-dotenv-file:
 	@ sed -i '/^RML_MAPPER_PATH/d' .env
 	@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
 
-refresh-mapping-files:
-	@ python -m ted_sws.data_manager.entrypoints.cli.cmd_generate_mapping_resources
-
 #clean-mongo-db:
 #	@ export PYTHONPATH=$(PWD) && python ./tests/clean_mongo_db.py
 

dags/dags_utils.py

@@ -1,17 +1,10 @@
-from itertools import islice, chain
-from typing import Any, Iterable
+from typing import Any
 
 from airflow.operators.python import get_current_context
 
 TASK_INSTANCE = "ti"
 
 
-def chunks(iterable: Iterable, chunk_size: int):
-    iterator = iter(iterable)
-    for first in iterator:
-        yield chain([first], islice(iterator, chunk_size - 1))
-
-
 def select_first_non_none(data):
     """
 

dags/fetch_notices_by_date.py

@@ -64,8 +64,7 @@ def validate_fetched_notices():
                                                            default_value=(datetime.now() - timedelta(days=1)).strftime(
                                                                "%Y%m%d*")), "%Y%m%d*")
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
-        validate_and_update_daily_supra_notice(notice_publication_day=publication_date,
-                                               mongodb_client=mongodb_client)
+        validate_and_update_daily_supra_notice(ted_publication_date=publication_date, mongodb_client=mongodb_client)
 
     def _branch_selector():
         trigger_complete_workflow = get_dag_param(key=TRIGGER_COMPLETE_WORKFLOW_DAG_KEY,

dags/operators/DagBatchPipelineOperator.py

@@ -4,8 +4,9 @@
 from airflow.operators.trigger_dagrun import TriggerDagRunOperator
 from pymongo import MongoClient
 
-from dags.dags_utils import pull_dag_upstream, push_dag_downstream, chunks, get_dag_param, smart_xcom_pull, \
+from dags.dags_utils import pull_dag_upstream, push_dag_downstream, get_dag_param, smart_xcom_pull, \
     smart_xcom_push
+from ted_sws.core.service.batch_processing import chunks
 from dags.pipelines.pipeline_protocols import NoticePipelineCallable
 from ted_sws import config
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository

dags/reprocess_unpublished_notices_from_backlog.py

@@ -14,7 +14,7 @@
 DAG_NAME = "reprocess_unpublished_notices_from_backlog"
 
 RE_PUBLISH_TARGET_NOTICE_STATES = [NoticeStatus.ELIGIBLE_FOR_PUBLISHING, NoticeStatus.INELIGIBLE_FOR_PUBLISHING,
-                                   NoticeStatus.PACKAGED
+                                   NoticeStatus.PACKAGED, NoticeStatus.PUBLICLY_UNAVAILABLE
                                    ]
 TRIGGER_NOTICE_PROCESS_WORKFLOW_TASK_ID = "trigger_notice_process_workflow"
 FORM_NUMBER_DAG_PARAM = "form_number"

infra/airflow-cluster/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/apache/airflow:2.4.3-python3.8
+FROM docker.io/apache/airflow:2.5.1-python3.8
 
 # quick sudo
 USER root
@@ -26,4 +26,4 @@ RUN wget -c https://github.com/dice-group/LIMES/releases/download/1.7.9/limes.ja
 
 
 RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.4.3/constraints-no-providers-3.8.txt"
+RUN pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt"

infra/airflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/apache/airflow:2.4.3-python3.8
+FROM docker.io/apache/airflow:2.5.1-python3.8
 
 # quick sudo
 USER root
@@ -25,4 +25,4 @@ RUN wget -c https://github.com/dice-group/LIMES/releases/download/1.7.9/limes.ja
 
 
 RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.4.3/constraints-no-providers-3.8.txt"
+RUN pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt"

infra/digest_api/digest_service/requirements.txt

@@ -1 +1 @@
-uvicorn[standard]
+uvicorn[standard]