Merge pull request #366 from OP-TED/feature/TED-677

Author: kaleanych

ted_sws/notice_metadata_processor/model/metadata.py

@@ -15,6 +15,7 @@ class ExtractedMetadata(Metadata):
     city_of_buyer: List[LanguageTaggedString] = None
     name_of_buyer: List[LanguageTaggedString] = None
     original_language: str = None
+    uri_list: str = None
     country_of_buyer: str = None
     type_of_buyer: EncodedValue = None
     eu_institution: str = None

ted_sws/notice_metadata_processor/services/xml_manifestation_metadata_extractor.py

@@ -86,6 +86,17 @@ def name_of_buyer(self):
     def eu_institution(self):
         return self.type_of_buyer.value if self.type_of_buyer.code == "5" else "-"
 
+    @property
+    def uri_list(self):
+        uri_elements = self.manifestation_root.findall(
+            self.xpath_registry.xpath_uri_elements,
+            namespaces=self.namespaces)
+
+        return [LanguageTaggedString(text=extract_text_from_element(element=uri.find(".")),
+                                     language=extract_attribute_from_element(element=uri.find("."),
+                                                                             attrib_key="LG")) for
+                uri in uri_elements]
+
     @property
     def country_of_buyer(self):
         return extract_attribute_from_element(element=self.manifestation_root.find(
@@ -219,14 +230,16 @@ def to_metadata(self) -> ExtractedMetadata:
          Creating extracted metadata
         :return:
         """
-        metadata = ExtractedMetadata()
+        metadata: ExtractedMetadata = ExtractedMetadata()
         metadata.title = self.title
         metadata.notice_publication_number = self.notice_publication_number
         metadata.publication_date = self.publication_date
+        metadata.ojs_type = self.ojs_type
         metadata.ojs_issue_number = self.ojs_issue_number
         metadata.city_of_buyer = self.city_of_buyer
         metadata.name_of_buyer = self.name_of_buyer
         metadata.original_language = self.original_language
+        metadata.uri_list = self.uri_list
         metadata.country_of_buyer = self.country_of_buyer
         metadata.type_of_buyer = self.type_of_buyer
         metadata.eu_institution = self.eu_institution

ted_sws/notice_metadata_processor/services/xpath_registry.py

@@ -43,6 +43,10 @@ def xpath_name_of_buyer_elements(self):
     def xpath_country_of_buyer(self):
         return "manifestation_ns:CODED_DATA_SECTION/manifestation_ns:NOTICE_DATA/manifestation_ns:ISO_COUNTRY"
 
+    @property
+    def xpath_uri_elements(self):
+        return "manifestation_ns:CODED_DATA_SECTION/manifestation_ns:NOTICE_DATA/manifestation_ns:URI_LIST/"
+
     @property
     def xpath_original_language(self):
         return "manifestation_ns:CODED_DATA_SECTION/manifestation_ns:NOTICE_DATA/manifestation_ns:LG_ORIG"

ted_sws/notice_packager/entrypoints/cli/cmd_bulk_packager.py

@@ -11,17 +11,22 @@
 import base64
 import os
 from pathlib import Path
+from typing import List
 
 import click
+from pymongo import MongoClient
 
+from ted_sws import config
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner
 from ted_sws.core.model.manifestation import XMLManifestation
 from ted_sws.core.model.notice import Notice
+from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
+from ted_sws.event_manager.adapters.log import LOG_WARN_TEXT
 from ted_sws.notice_metadata_processor.services.xml_manifestation_metadata_extractor import \
     XMLManifestationMetadataExtractor
-from ted_sws.notice_packager.services.metadata_transformer import MetadataTransformer
-from ted_sws.notice_packager.services.notice_packager import create_notice_package
 from ted_sws.notice_packager import DEFAULT_NOTICE_PACKAGE_EXTENSION
+from ted_sws.notice_packager.services.metadata_transformer import MetadataTransformer
+from ted_sws.notice_packager.services.notice_packager import create_notice_package, package_notice_and_save_to
 
 CMD_NAME = "CMD_BULK_PACKAGER"
 DEFAULT_FILES_COUNT: int = 3000
@@ -35,47 +40,60 @@ class PackageNotice(Notice):
 
 
 class CmdRunner(BaseCmdRunner):
-    def __init__(self, rdf_files_folder, output_folder, pkgs_count: int):
+    def __init__(self, rdf_files_folder, output_folder, pkgs_count: int, notice_ids: List = None,
+                 mongodb_client=MongoClient(config.MONGO_DB_AUTH_URL)):
         super().__init__(name=CMD_NAME)
-        self.rdf_files_path = Path(os.path.realpath(rdf_files_folder))
         self.output_path = Path(os.path.realpath(output_folder))
-        self.pkgs_count = pkgs_count
-        if not self.rdf_files_path.is_dir():
-            error_msg = f"No such folder :: [{rdf_files_folder}]"
-            self.log_failed_msg(error_msg)
-            raise FileNotFoundError(error_msg)
+        self.notices = None
+        if notice_ids:
+            self.log(LOG_WARN_TEXT.format("Notices: ") + str(notice_ids))
+            self.notice_repository = NoticeRepository(mongodb_client=mongodb_client)
+            self.notices = []
+            for notice_id in notice_ids:
+                self.notices.append(self.notice_repository.get(reference=notice_id))
+        else:
+            self.rdf_files_path = Path(os.path.realpath(rdf_files_folder))
+            self.pkgs_count = pkgs_count
+            if not self.rdf_files_path.is_dir():
+                error_msg = f"No such folder :: [{rdf_files_folder}]"
+                self.log_failed_msg(error_msg)
+                raise FileNotFoundError(error_msg)
+
+        self.output_path.mkdir(parents=True, exist_ok=True)
 
     def run_cmd(self):
         error = None
         try:
-            self.output_path.mkdir(parents=True, exist_ok=True)
-            rdf_files = [Path(str(f_path)) for f in os.listdir(self.rdf_files_path) if
-                         os.path.isfile(f_path := os.path.join(self.rdf_files_path, f))]
-            rdf_files_count = len(rdf_files)
-            base_idx = 100000
-            year = 2021
-
-            for i in range(self.pkgs_count):
-                rdf_idx = i % rdf_files_count
-                rdf_file_path = rdf_files[rdf_idx]
-                notice_id = str(base_idx + i) + "_" + str(year)
-                pkg_name = notice_id
-                self.generate_package(notice_id, self.output_path, rdf_file_path, pkg_name)
+            if self.notices:
+                self.log("Saving packages to " + str(self.output_path))
+                for notice in self.notices:
+                    package_notice_and_save_to(notice=notice,
+                                               save_to=self.output_path)
+            else:
+                rdf_files = [Path(str(f_path)) for f in os.listdir(self.rdf_files_path) if
+                             os.path.isfile(f_path := os.path.join(self.rdf_files_path, f))]
+                rdf_files_count = len(rdf_files)
+                base_idx = 100000
+                year = 2021
+
+                for i in range(self.pkgs_count):
+                    rdf_idx = i % rdf_files_count
+                    rdf_file_path = rdf_files[rdf_idx]
+                    notice_id = str(base_idx + i) + "_" + str(year)
+                    self.generate_package(notice_id, self.output_path, rdf_file_path)
         except Exception as e:
             error = e
 
         return self.run_cmd_result(error)
 
     @classmethod
-    def generate_package(cls, notice_id, output_path, rdf_file_path, pkg_name):
+    def generate_package(cls, notice_id, output_path, rdf_file_path):
 
         with open(rdf_file_path, "r") as f:
             rdf_content = f.read()
 
             encoded_rdf_content = base64.b64encode(bytes(rdf_content, 'utf-8'))
 
-            output_file = output_path / (pkg_name + DEFAULT_NOTICE_PACKAGE_EXTENSION)
-
             notice = PackageNotice(ted_id=notice_id)
             notice_metadata = XMLManifestationMetadataExtractor(
                 xml_manifestation=notice.xml_manifestation).to_metadata()
@@ -84,24 +102,26 @@ def generate_package(cls, notice_id, output_path, rdf_file_path, pkg_name):
             create_notice_package(
                 notice_metadata,
                 rdf_content=encoded_rdf_content,
-                save_to=output_file
+                save_to=output_path
             )
 
 
-def run(rdf_files_count, output_folder, pkgs_count):
-    cmd = CmdRunner(rdf_files_count, output_folder, pkgs_count)
+def run(rdf_files_count=None, output_folder=None, pkgs_count=None, notice_id=None,
+        mongodb_client=MongoClient(config.MONGO_DB_AUTH_URL)):
+    cmd = CmdRunner(rdf_files_count, output_folder, pkgs_count, list(notice_id or []), mongodb_client)
     cmd.run()
 
 
 @click.command()
-@click.argument('rdf-files-folder', nargs=1)
-@click.argument('output-folder', nargs=1)
+@click.argument('rdf-files-folder', nargs=1, required=False)
 @click.argument('pkgs-count', nargs=1, type=click.INT, required=False, default=DEFAULT_FILES_COUNT)
-def main(rdf_files_folder, output_folder, pkgs_count):
+@click.option('--output-folder', required=False, default=".")
+@click.option('--notice-id', required=False, multiple=True, default=None)
+def main(rdf_files_folder, pkgs_count, output_folder, notice_id):
     """
-    Generates <PKGS_COUNT> test METS packages
+    Generates test METS packages
     """
-    run(rdf_files_folder, output_folder, pkgs_count)
+    run(rdf_files_folder, output_folder, pkgs_count, notice_id)
 
 
 if __name__ == '__main__':

ted_sws/notice_packager/model/metadata.py

@@ -10,7 +10,7 @@
 """
 
 import datetime
-from typing import List, Dict
+from typing import List, Dict, Optional
 
 from pydantic import validator
 
@@ -28,12 +28,12 @@
 BASE_TITLE = "eProcurement notice"
 
 WORK_DO_NOT_INDEX = "true"
-MANIFESTATION_TYPE = "E_PROCUREMENT_ONTOLOGY"
+MANIFESTATION_TYPE = "rdf_epo"
 DISTRIBUTION_STATUS = "COMPLETED"
 MEDIA_TYPE = "RDF"
 LANGUAGES = ["en"]
 LANGUAGE = LANGUAGES[0]
-USES_LANGUAGE = "ENG"
+USES_LANGUAGE = "MUL"
 
 ACTION_CREATE = "create"
 ACTION_UPDATE = "update"
@@ -64,7 +64,7 @@ class NoticeMetadata(Metadata):
     """
     General notice metadata
     """
-    id: str = None
+    id: Optional[str] = None
     languages: List[str] = LANGUAGES
     action: NoticeActionMetadata = NoticeActionMetadata()
 
@@ -75,22 +75,27 @@ class WorkMetadata(Metadata):
         and the rest is a bunch of constants OR generated values (e.g. date, URI, ...)
     """
 
-    uri: str = None
+    identifier: Optional[str]
+    cdm_rdf_type: Optional[str]
+    resource_type: Optional[str]
+    uri: Optional[str] = None
     do_not_index: str = WORK_DO_NOT_INDEX
     date_document: str = datetime.datetime.now().strftime('%Y-%m-%d')
     created_by_agent: str = WORK_AGENT
     dataset_published_by_agent: str = WORK_AGENT
     datetime_transmission: str = datetime.datetime.now().isoformat()
-    title: Dict[str, str] = None
-    date_creation: str = None
+    title: Optional[Dict[str, str]] = None
+    date_creation: Optional[str] = datetime.datetime.now().strftime('%Y-%m-%d')
     concept_type_dataset: str = CONCEPT_TYPE_DATASET
-    dataset_version: str = None
+    dataset_version: Optional[str] = None
     dataset_keyword: List[str] = DATASET_KEYWORD
     dataset_has_frequency_publication_frequency: str = PUBLICATION_FREQUENCY
+    procurement_public_issued_by_country: Optional[str]
+    procurement_public_url_etendering: Optional[List[str]]
 
 
 class ExpressionMetadata(Metadata):
-    title: Dict[str, str] = None
+    title: Optional[Dict[str, str]] = None
     uses_language: str = USES_LANGUAGE
 
 

ted_sws/notice_packager/resources/templates/mets2action_mets_xml.jinja2

@@ -6,7 +6,7 @@
       TYPE="{{ notice.action.type }}"
       PROFILE="http://publications.europa.eu/resource/mets/op-sip-profile_002">
     <metsHdr {{ notice.action.type|upper }}DATE="{{ notice.action.date }}">
-        <metsDocumentID>{{ notice.id }}_mets2{{ notice.action.type }}</metsDocumentID>
+        <metsDocumentID>{{ work.identifier }}_{{ notice.action.type }}</metsDocumentID>
     </metsHdr>
     <dmdSec ID="dmdSec01">
         <mdRef MDTYPE="OTHER" LOCTYPE="URL" MIMETYPE="application/rdf+xml" OTHERMDTYPE="INSTANCE" xlink:href="{{ notice.id }}-0.mets.xml.dmd.rdf"/>

ted_sws/notice_packager/resources/templates/mets_xml_dmd_rdf.jinja2

@@ -9,39 +9,46 @@
          xmlns:dct="http://purl.org/dc/terms/"
          xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
 
-    <cdm:work rdf:about="&resource;dataset/{{ notice.id }}">
+    <cdm:work rdf:about="&resource;ted/{{ work.identifier }}">
+        <rdf:type rdf:resource="http://publications.europa.eu/ontology/cdm#work"/>
+        <rdf:type rdf:resource="http://publications.europa.eu/ontology/cdm#procurement_public"/>
+        <cdm:work_has_resource-type rdf:resource="http://publications.europa.eu/resource/authority/resource-type/PROCUREMENT_NOTICE"/>
         <cdm:do_not_index rdf:datatype="http://www.w3.org/2001/XMLSchema#boolean">{{ work.do_not_index }}</cdm:do_not_index>
         <cdm:work_date_document rdf:datatype="http://www.w3.org/2001/XMLSchema#date">{{ work.date_document }}</cdm:work_date_document>
         <cdm:work_created_by_agent rdf:resource="&cellar-authority;corporate-body/{{ work.created_by_agent }}"/>
-        <cdm:work_dataset_published_by_agent rdf:resource="&cellar-authority;corporate-body/{{ work.dataset_published_by_agent }}"/>
-        <cdm:datetime_transmission rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">{{ work.datetime_transmission }}</cdm:datetime_transmission>
-        <cdm:datetime_negotiation rdf:resource="http://publications.europa.eu/ontology/cdm#datetime_transmission"/>
         {% for lang in notice.languages %}
         <cdm:work_title xml:lang="{{ lang }}">{{ work.title[lang] }}</cdm:work_title>
         {% endfor %}
+        <cdm:datetime_transmission rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">{{ work.datetime_transmission }}</cdm:datetime_transmission>
+        {# <cdm:procurement_public_issued_by_country>{{ work.procurement_public_issued_by_country }}</cdm:procurement_public_issued_by_country>
+        {% for uri in work.procurement_public_url_etendering %}
+        <cdm:procurement_public_url_etendering xml:lang="{{ uri.language }}">{{ uri.text }}</cdm:procurement_public_url_etendering>
+        {% endfor %} #}
+        {# <cdm:datetime_negotiation rdf:resource="http://publications.europa.eu/ontology/cdm#datetime_transmission"/>
+        <cdm:work_dataset_published_by_agent rdf:resource="&cellar-authority;corporate-body/{{ work.dataset_published_by_agent }}"/>
         <cdm:work_date_creation rdf:datatype="http://www.w3.org/2001/XMLSchema#date">{{ work.date_creation }}</cdm:work_date_creation>
         <cdm:work_id>{{ work.uri }}</cdm:work_id>
         <cdm:work_dataset_version>{{ work.dataset_version }}</cdm:work_dataset_version>
         <cdm:work_dataset_has_type_concept_type_dataset rdf:resource="http://publications.europa.eu/resource/authority/dataset-type/{{ work.concept_type_dataset }}"/>
         {% for dataset_keyword in work.dataset_keyword %}
         <cdm:work_dataset_keyword>{{ dataset_keyword }}</cdm:work_dataset_keyword>
         {% endfor %}
-        <cdm:work_dataset_has_frequency_publication_frequency rdf:resource="http://publications.europa.eu/resource/authority/frequency/{{ work.dataset_has_frequency_publication_frequency }}"/>
+        <cdm:work_dataset_has_frequency_publication_frequency rdf:resource="http://publications.europa.eu/resource/authority/frequency/{{ work.dataset_has_frequency_publication_frequency }}"/> #}
     </cdm:work>
 
-    <cdm:expression rdf:about="&resource;expression/{{ notice.id }}">
-        <cdm:expression_belongs_to_work rdf:resource="&resource;dataset/{{ notice.id }}"/>
+    <cdm:expression rdf:about="&resource;ted/{{ work.identifier }}.MUL">
+        <cdm:expression_belongs_to_work rdf:resource="&resource;ted/{{ work.identifier }}"/>
         {% for lang in notice.languages %}
         <cdm:expression_title xml:lang="{{ lang }}">{{ expression.title[lang] }}</cdm:expression_title>
         {% endfor %}
         <cdm:expression_uses_language rdf:resource="&cellar-authority;language/{{ expression.uses_language }}"/>
     </cdm:expression>
 
-    <cdm:manifestation_distribution rdf:about="&resource;distribution/{{ notice.id }}/{{ notice.id }}_rdf">
-        <cdm:manifestation_manifests_expression rdf:resource="&resource;expression/{{ notice.id }}"/>
+    <cdm:manifestation_distribution rdf:about="&resource;ted/{{ work.identifier }}.MUL.rdf">
+        <cdm:manifestation_manifests_expression rdf:resource="&resource;ted/{{ work.identifier }}.MUL"/>
         <cdm:manifestation_type rdf:datatype="http://www.w3.org/2001/XMLSchema#string">{{ manifestation.type }}</cdm:manifestation_type>
         <cdm:manifestation_date_publication rdf:datatype="http://www.w3.org/2001/XMLSchema#date">{{ manifestation.date_publication }}</cdm:manifestation_date_publication>
-        <cdm:manifestation_distribution_has_status_distribution_status rdf:resource="http://publications.europa.eu/resource/authority/dataset-status/{{ manifestation.distribution_has_status_distribution_status }}"/>
-        <cdm:manifestation_distribution_has_media_type_concept_media_type rdf:resource="http://publications.europa.eu/resource/authority/file-type/{{ manifestation.distribution_has_media_type_concept_media_type }}"/>
+        {# <cdm:manifestation_distribution_has_status_distribution_status rdf:resource="http://publications.europa.eu/resource/authority/dataset-status/{{ manifestation.distribution_has_status_distribution_status }}"/>
+        <cdm:manifestation_distribution_has_media_type_concept_media_type rdf:resource="http://publications.europa.eu/resource/authority/file-type/{{ manifestation.distribution_has_media_type_concept_media_type }}"/> #}
     </cdm:manifestation_distribution>
 </rdf:RDF>

ted_sws/notice_packager/services/metadata_transformer.py

@@ -23,6 +23,9 @@
 # This is used in TED API
 DENORMALIZED_SEPARATOR = '-'
 
+PROCUREMENT_PUBLIC = "procurement_public"
+PROCUREMENT_NOTICE = "PROCUREMENT_NOTICE"
+
 
 class MetadataTransformer:
     def __init__(self, notice_metadata: ExtractedMetadata):
@@ -64,23 +67,40 @@ def from_notice_metadata(cls, notice_metadata: ExtractedMetadata) -> PackagerMet
         metadata.notice.id = cls.normalize_value(notice_metadata.notice_publication_number)
 
         # WORK
-        metadata.work.uri = publication_notice_uri(metadata.notice.id)
+        publication_date = datetime.datetime.strptime(notice_metadata.publication_date, '%Y%m%d').strftime('%Y-%m-%d')
+        metadata.work.identifier = publication_work_identifier(metadata.notice.id, notice_metadata)
+        metadata.work.cdm_rdf_type = PROCUREMENT_PUBLIC
+        metadata.work.resource_type = PROCUREMENT_NOTICE
+        metadata.work.date_document = publication_date
+        metadata.work.uri = publication_notice_uri(metadata.notice.id, notice_metadata)
         title_search = [t.title.text for t in notice_metadata.title if t.title.language == LANGUAGE.upper()]
         if len(title_search) > 0:
             metadata.work.title = {LANGUAGE: title_search[0]}
-        metadata.work.date_creation = datetime.datetime \
-            .strptime(notice_metadata.publication_date, '%Y%m%d').strftime('%Y-%m-%d')
         metadata.work.dataset_version = _date.strftime('%Y%m%d') + '-' + _revision
+        metadata.work.procurement_public_issued_by_country = notice_metadata.country_of_buyer
+        metadata.work.procurement_public_url_etendering = notice_metadata.uri_list
 
         # EXPRESSION
         metadata.expression.title = {LANGUAGE: BASE_TITLE + " " + metadata.notice.id}
 
+        # MANIFESTATION
+        metadata.manifestation.date_publication = publication_date
         return metadata
 
 
-def publication_notice_year(notice_id):
-    return notice_id.split(NORMALIZED_SEPARATOR)[1]
+def publication_notice_year(notice_metadata):
+    return datetime.datetime.strptime(notice_metadata.publication_date, '%Y%m%d').strftime('%Y')
+
+
+def publication_notice_number(notice_id):
+    return notice_id.split(NORMALIZED_SEPARATOR)[0]
+
+
+def publication_notice_uri(notice_id, notice_metadata):
+    return f"{BASE_WORK}{publication_notice_year(notice_metadata)}/{notice_id}"
 
 
-def publication_notice_uri(notice_id):
-    return f"{BASE_WORK}{publication_notice_year(notice_id)}/{notice_id}"
+def publication_work_identifier(notice_id, notice_metadata):
+    year = publication_notice_year(notice_metadata)
+    number = publication_notice_number(notice_id)
+    return f"{year}_{notice_metadata.ojs_type}_{notice_metadata.ojs_issue_number}_{number}"