Merge branch 'feature/TED-1168' into feature/TED-1191

Author: CaptainOfHacks

.github/workflows/unit-tests-srv.yml

@@ -3,9 +3,14 @@ name: Tests on srv server
 on:
   workflow_dispatch:
   push:
-    branches: [ feature/* , main,  hotfix/* ]
-  pull_request:
-    branches: [ main, release/* ]
+    branches: [ feature/* , main,  hotfix/*, release/* ]
+#  pull_request:
+#    branches: [ main, release/* ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
   VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
@@ -30,11 +35,11 @@ jobs:
       - name: Get RML mapper
         run: make init-rml-mapper
 
-#      - name: Get Allure
-#        run: make install-allure
+      #      - name: Get Allure
+      #        run: make install-allure
 
-#      - name: Start staging infra
-#        run: make start-project-services
+      #      - name: Start staging infra
+      #        run: make start-project-services
 
       - name: Run tests
         run: make test-all

.github/workflows/unit-tests.yml

@@ -2,35 +2,39 @@ name: Tests
 
 on:
   push:
-    branches: [ feature/* , main, hotfix/* ]
-  pull_request:
-    branches: [ main, release/* ]
+    branches: [ feature/* , main, hotfix/*, release/* ]
+#  pull_request:
+#    branches: [ main, release/* ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
     name: pyTest
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade setuptools pip wheel
-        make install
-        make install-dev
-    - name: Make envfile
-      uses: SpicyPizza/create-envfile@v1
-      with:
-        envkey_VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
-        envkey_VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
-    - name: Run unit tests
-      run: make test
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade setuptools pip wheel
+          make install
+          make install-dev
+      - name: Make envfile
+        uses: SpicyPizza/create-envfile@v1
+        with:
+          envkey_VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
+          envkey_VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+      - name: Run unit tests
+        run: make test
 #    - name: Upload coverage to Codecov
 #      uses: codecov/codecov-action@v1
 #      with:

dags/fetch_notices_by_date.py

@@ -64,8 +64,7 @@ def validate_fetched_notices():
                                                            default_value=(datetime.now() - timedelta(days=1)).strftime(
                                                                "%Y%m%d*")), "%Y%m%d*")
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
-        validate_and_update_daily_supra_notice(notice_publication_day=publication_date,
-                                               mongodb_client=mongodb_client)
+        validate_and_update_daily_supra_notice(ted_publication_date=publication_date, mongodb_client=mongodb_client)
 
     def _branch_selector():
         trigger_complete_workflow = get_dag_param(key=TRIGGER_COMPLETE_WORKFLOW_DAG_KEY,

dags/reprocess_unpublished_notices_from_backlog.py

@@ -14,7 +14,7 @@
 DAG_NAME = "reprocess_unpublished_notices_from_backlog"
 
 RE_PUBLISH_TARGET_NOTICE_STATES = [NoticeStatus.ELIGIBLE_FOR_PUBLISHING, NoticeStatus.INELIGIBLE_FOR_PUBLISHING,
-                                   NoticeStatus.PACKAGED
+                                   NoticeStatus.PACKAGED, NoticeStatus.PUBLICLY_UNAVAILABLE
                                    ]
 TRIGGER_NOTICE_PROCESS_WORKFLOW_TASK_ID = "trigger_notice_process_workflow"
 FORM_NUMBER_DAG_PARAM = "form_number"

infra/digest_api/digest_service/requirements.txt

@@ -1 +1 @@
-uvicorn[standard]
+uvicorn[standard]

infra/fuseki/docker-compose.yml

@@ -9,7 +9,8 @@ services:
       - ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD}
       - FUSEKI_DATASET_1=mydataset
     volumes:
-      - fuseki-data:/fuseki-data/databases
+      - fuseki-data:/fuseki-base/databases
+      - fuseki-data:/fuseki-base/configuration
     networks:
       - fuseki-net
       - proxy-net

requirements.txt

@@ -5,7 +5,7 @@ deepdiff~=5.7.0
 jinja2~=3.1.2
 python-dotenv~=0.19.2
 pymongo~=4.0.1
-apache-airflow==2.4.3
+apache-airflow~=2.5.1
 hvac==0.11.2
 SPARQLWrapper==1.8.5
 pandas==1.5.1
@@ -25,4 +25,5 @@ ordered-set~=4.0.2
 json2html~=1.3.0
 minio~=7.1.1
 certifi
-networkx~=2.8.8
+networkx~=2.8.8
+shortuuid~=1.0.11

sonar-project.properties

@@ -23,7 +23,6 @@ sonar.links.scm=https://github.com/OP-TED/ted-rdf-conversion-pipeline
 sonar.links.issue=https://github.com/OP-TED/ted-rdf-conversion-pipeline/issues
 
 sonar.tests=tests
-# sonar.login=67bcbf1aefb992993fdd45b78fb4977a05be39b9
 # sonar.host.url=https://sonarcloud.io
 
 sonar.python.pylint=/usr/local/bin/pylint

ted_sws/core/model/lazy_object.py

@@ -0,0 +1,62 @@
+import abc
+from typing import Optional, Any
+
+
+class LazyObjectFieldsLoaderABC(abc.ABC):
+
+    @abc.abstractmethod
+    def load_lazy_field(self, source_object: Any, property_field: property) -> Any:
+        """
+
+        :param source_object:
+        :param property_field:
+        :return:
+        """
+
+    @abc.abstractmethod
+    def remove_lazy_field(self, source_object: Any, property_field: property):
+        """
+
+        :param source_object:
+        :param property_field:
+        :return:
+        """
+
+
+class LazyObjectABC(abc.ABC):
+
+    @abc.abstractmethod
+    def set_lazy_object_fields_loader(self, lazy_object_fields_loader: LazyObjectFieldsLoaderABC):
+        """
+
+        :param lazy_object_fields_loader:
+        :return:
+        """
+
+    @abc.abstractmethod
+    def get_lazy_object_fields_loader(self) -> Optional[LazyObjectFieldsLoaderABC]:
+        """
+
+        :return:
+        """
+
+    def load_lazy_field(self, property_field: property):
+        """
+
+        :param property_field:
+        :return:
+        """
+        if self.get_lazy_object_fields_loader():
+            return self.get_lazy_object_fields_loader().load_lazy_field(source_object=self,
+                                                                        property_field=property_field)
+        return None
+
+    def remove_lazy_field(self, property_field: property):
+        """
+
+        :param property_field:
+        :return:
+        """
+        if self.get_lazy_object_fields_loader():
+            self.get_lazy_object_fields_loader().remove_lazy_field(source_object=self,
+                                                                   property_field=property_field)

ted_sws/core/model/metadata.py

@@ -82,6 +82,31 @@ class NormalisedMetadata(Metadata):
     xsd_version: str
 
 
+class NormalisedMetadataView(Metadata):
+    title: str
+    long_title: str
+    notice_publication_number: str
+    publication_date: str
+    ojs_issue_number: str
+    ojs_type: str
+    city_of_buyer: Optional[str]
+    name_of_buyer: Optional[str]
+    original_language: Optional[str]
+    country_of_buyer: Optional[str]
+    eu_institution: Optional[bool]
+    document_sent_date: Optional[str]
+    deadline_for_submission: Optional[str]
+    notice_type: str
+    form_type: str
+    place_of_performance: Optional[List[str]]
+    extracted_legal_basis_directive: Optional[str]
+    legal_basis_directive: str
+    form_number: str
+    eforms_subtype: str
+    xsd_version: str
+
+
+
 class TEDMetadata(Metadata):
     """
         Stores notice original metadata