Skip to content

Commit d7d03a6

Browse files
kaleanychkaleanych
committed
METS generation updates + tests
1 parent 2a4a48c commit d7d03a6

20 files changed

Lines changed: 575 additions & 3107 deletions

‎ted_sws/notice_packager/adapters/template_generator.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
from jinja2 import Environment, PackageLoader
1313

14-
from ted_sws.notice_packager.model.metadata import PackagerMetadata, validate_notice_action_type
14+
from ted_sws.notice_packager.model.metadata import PackagerMetadata, validate_mets_type
1515

1616
TEMPLATES = Environment(loader=PackageLoader("ted_sws.notice_packager.resources", "templates"))
1717

@@ -34,8 +34,8 @@ def tmd_rdf_generator(cls, data: PackagerMetadata = None) -> str:
3434

3535
@classmethod
3636
def mets2action_mets_xml_generator(cls, data: PackagerMetadata = None) -> str:
37-
action = data.notice.action.type
38-
validate_notice_action_type(action)
37+
action = data.mets.type
38+
validate_mets_type(action)
3939

4040
template = 'mets2action_mets_xml.jinja2'
4141
return cls.__generate_template(template, data)

‎ted_sws/notice_packager/model/metadata.py‎

Lines changed: 54 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,25 @@
1616

1717
from ted_sws.core.model.metadata import Metadata
1818

19-
WORK_AGENT = "PUBL"
19+
METS_PROFILE = "http://publications.europa.eu/resource/mets/op-sip-profile_002"
20+
METS_TYPE_CREATE = "create"
21+
METS_TYPE_UPDATE = "update"
22+
METS_TYPE_DELETE = "delete"
23+
METS_ACCEPTED_TYPES = [METS_TYPE_CREATE, METS_TYPE_UPDATE, METS_TYPE_DELETE]
24+
METS_DMD_MDTYPE = "OTHER"
25+
METS_DMD_OTHERMDTYPE = "INSTANCE"
26+
METS_DMD_HREF = "{work_identifier}_{revision}.mets.xml.dmd.rdf"
27+
METS_DMD_ID = "{work_identifier}_{revision}_dmd_{dmd_idx}"
28+
METS_TMD_ID = "{work_identifier}_{revision}_tmd_{tmd_idx}"
29+
METS_TMD_HREF = "{work_identifier}_{revision}.tmd.rdf"
30+
METS_TMD_MDTYPE = "OTHER"
31+
METS_TMD_OTHERMDTYPE = "INSTANCE"
32+
METS_FILE_ID = "{work_identifier}_{revision}_file_{file_idx}"
33+
METS_NOTICE_FILE_HREF = "{work_identifier}_{revision}.notice.ttl"
34+
METS_NOTICE_FILE_MIMETYPE = "text/turtle"
35+
METS_NOTICE_FILE_CHECKSUM_TYPE = "SHA-256"
36+
37+
WORK_AGENT = "EURUN"
2038
PUBLICATION_FREQUENCY = "OTHER"
2139
CONCEPT_TYPE_DATASET = "TEST_DATA"
2240
DATASET_KEYWORD = [
@@ -35,38 +53,51 @@
3553
LANGUAGE = LANGUAGES[0]
3654
USES_LANGUAGE = "MUL"
3755

38-
ACTION_CREATE = "create"
39-
ACTION_UPDATE = "update"
40-
ACCEPTED_ACTIONS = [ACTION_CREATE, ACTION_UPDATE]
41-
4256
REVISION = "0"
4357

4458

45-
def validate_notice_action_type(v):
46-
if v not in ACCEPTED_ACTIONS:
47-
raise ValueError('No such action: %s' % v)
59+
def validate_mets_type(v):
60+
if v not in METS_ACCEPTED_TYPES:
61+
raise ValueError('No such METS type: %s' % v)
4862

4963

50-
class NoticeActionMetadata(Metadata):
64+
class NoticeMetadata(Metadata):
5165
"""
52-
Notice action metadata
66+
General notice metadata
5367
"""
54-
type: str = ACTION_CREATE
55-
date: str = datetime.datetime.now().isoformat()
56-
57-
@validator('type')
58-
def validate_notice_action_type(cls, v):
59-
validate_notice_action_type(v)
60-
return v
68+
id: Optional[str]
69+
number: Optional[str]
6170

6271

63-
class NoticeMetadata(Metadata):
72+
class MetsMetadata(Metadata):
6473
"""
6574
General notice metadata
6675
"""
67-
id: Optional[str] = None
6876
languages: List[str] = LANGUAGES
69-
action: NoticeActionMetadata = NoticeActionMetadata()
77+
revision: str = REVISION
78+
79+
type: str = METS_TYPE_CREATE
80+
profile: str = METS_PROFILE
81+
createdate: str = datetime.datetime.now().isoformat()
82+
document_id: Optional[str]
83+
dmd_id: Optional[str]
84+
dmd_mdtype: str = METS_DMD_MDTYPE
85+
dmd_othermdtype: str = METS_DMD_OTHERMDTYPE
86+
dmd_href: Optional[str]
87+
tmd_id: Optional[str]
88+
tmd_href: Optional[str]
89+
tmd_mdtype: str = METS_TMD_MDTYPE
90+
tmd_othermdtype: str = METS_TMD_OTHERMDTYPE
91+
file_id: Optional[str]
92+
notice_file_href: Optional[str]
93+
notice_file_mimetype: Optional[str] = METS_NOTICE_FILE_MIMETYPE
94+
notice_file_checksum: Optional[str]
95+
notice_file_checksum_type: Optional[str] = METS_NOTICE_FILE_CHECKSUM_TYPE
96+
97+
@validator('type')
98+
def validate_notice_action_type(cls, v):
99+
validate_mets_type(v)
100+
return v
70101

71102

72103
class WorkMetadata(Metadata):
@@ -95,11 +126,13 @@ class WorkMetadata(Metadata):
95126

96127

97128
class ExpressionMetadata(Metadata):
129+
identifier: Optional[str]
98130
title: Optional[Dict[str, str]] = None
99131
uses_language: str = USES_LANGUAGE
100132

101133

102134
class ManifestationMetadata(Metadata):
135+
identifier: Optional[str]
103136
type: str = MANIFESTATION_TYPE
104137
date_publication: str = datetime.datetime.now().strftime('%Y-%m-%d')
105138
distribution_has_status_distribution_status: str = DISTRIBUTION_STATUS
@@ -108,6 +141,7 @@ class ManifestationMetadata(Metadata):
108141

109142
class PackagerMetadata(Metadata):
110143
notice: NoticeMetadata = NoticeMetadata()
144+
mets: MetsMetadata = MetsMetadata()
111145
work: WorkMetadata = WorkMetadata()
112146
expression: ExpressionMetadata = ExpressionMetadata()
113147
manifestation: ManifestationMetadata = ManifestationMetadata()

‎ted_sws/notice_packager/resources/templates/mets2action_mets_xml.jinja2‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,38 +2,38 @@
22
<mets xmlns="http://www.loc.gov/METS/"
33
xmlns:xlink="http://www.w3.org/1999/xlink"
44
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5-
xsi:schemaLocation="http://www.loc.gov/METS/ cellar-mets.xsd"
6-
TYPE="{{ notice.action.type }}"
7-
PROFILE="http://publications.europa.eu/resource/mets/op-sip-profile_002">
8-
<metsHdr {{ notice.action.type|upper }}DATE="{{ notice.action.date }}">
9-
<metsDocumentID>{{ work.identifier }}_{{ notice.action.type }}</metsDocumentID>
5+
xsi:schemaLocation="http://www.loc.gov/METS/cellar-mets.xsd"
6+
TYPE="{{ mets.type }}"
7+
PROFILE="{{ mets.profile }}">
8+
<metsHdr CREATEDATE="{{ mets.createdate }}">
9+
<metsDocumentID>{{ mets.document_id }}</metsDocumentID>
1010
</metsHdr>
11-
<dmdSec ID="dmdSec01">
12-
<mdRef MDTYPE="OTHER" LOCTYPE="URL" MIMETYPE="application/rdf+xml" OTHERMDTYPE="INSTANCE" xlink:href="{{ notice.id }}-0.mets.xml.dmd.rdf"/>
11+
<dmdSec ID="{{ mets.dmd_id }}">
12+
<mdRef MDTYPE="{{ mets.dmd_mdtype }}" LOCTYPE="URL" MIMETYPE="application/rdf+xml" OTHERMDTYPE="{{ mets.dmd_othermdtype }}" xlink:href="{{ mets.dmd_href }}"/>
1313
</dmdSec>
1414
<amdSec>
15-
<techMD ID="techMDID001">
16-
<mdRef MDTYPE="OTHER" LOCTYPE="URL" MIMETYPE="application/rdf+xml" OTHERMDTYPE="INSTANCE" xlink:href="techMDID001.tmd.rdf"/>
15+
<techMD ID="{{ mets.tmd_id }}">
16+
<mdRef MDTYPE="{{ mets.tmd_mdtype }}" LOCTYPE="URL" MIMETYPE="application/rdf+xml" OTHERMDTYPE="{{ mets.tmd_othermdtype }}" xlink:href="{{ mets.tmd_href }}"/>
1717
</techMD>
1818
</amdSec>
1919
<fileSec>
2020
<fileGrp>
21-
<file ID="file-001" MIMETYPE="application/rdf+xml" CHECKSUM="f9cdda52af5e532068547f0c91fcf186840bd088 " CHECKSUMTYPE="SHA-1">
22-
<FLocat LOCTYPE="URL" xlink:href="{{ notice.id }}.rdf"/>
21+
<file ID="{{ mets.file_id }}" MIMETYPE="{{ mets.notice_file_mimetype }}" CHECKSUM="{{ mets.notice_file_checksum }}" CHECKSUMTYPE="{{ mets.notice_file_checksum_type }}">
22+
<FLocat LOCTYPE="URL" xlink:href="{{ mets.notice_file_href }}"/>
2323
</file>
2424
</fileGrp>
2525
</fileSec>
26-
<structMap ID="structMap01">
27-
<div TYPE="work" CONTENTIDS="dataset:{{ notice.id }}" DMDID="dmdSec01" ID="w-01">
28-
<div TYPE="expression" CONTENTIDS="expression:{{ notice.id }}" DMDID="dmdSec01" ID="e-01">
29-
<div TYPE="manifestation" CONTENTIDS="distribution:{{ notice.id }}/{{ notice.id }}_rdf" DMDID="dmdSec01" ADMID="techMDID001" ID="m-001">
30-
<fptr CONTENTIDS="distribution:{{ notice.id }}/{{ notice.id }}.rdf" FILEID="file-001"/>
26+
<structMap ID="{{ work.identifier }}_struct_map_001">
27+
<div TYPE="work" CONTENTIDS="ted:{{ work.identifier }}" DMDID="{{ mets.dmd_id }}" ID="{{ work.identifier }}_w_001">
28+
<div TYPE="expression" CONTENTIDS="ted:{{ expression.identifier }}" DMDID="{{ mets.dmd_id }}" ID="{{ work.identifier }}_e_001">
29+
<div TYPE="manifestation" CONTENTIDS="ted:{{ manifestation.identifier }}" DMDID="{{ mets.dmd_id }}" ADMID="{{ mets.tmd_id }}" ID="{{ work.identifier }}_m_001">
30+
<fptr CONTENTIDS="distribution:{{ notice.id }}/{{ mets.notice_file_href }}" FILEID="{{ mets.file_id }}"/>
3131
</div>
3232
</div>
3333
</div>
3434
</structMap>
3535
<behaviorSec>
36-
<behavior BTYPE="sparql-load" STRUCTID="m-001">
36+
<behavior BTYPE="sparql-load" STRUCTID="{{ work.identifier }}_m_001">
3737
<mechanism LOCTYPE="URL" LABEL="Sparql-load" xlink:href="cellar-mets:sparql-load-behavior?model={{ work.uri | urlencode | replace("/", "%2F") }}"/>
3838
</behavior>
3939
</behaviorSec>

‎ted_sws/notice_packager/resources/templates/mets_xml_dmd_rdf.jinja2‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@
1616
<cdm:do_not_index rdf:datatype="http://www.w3.org/2001/XMLSchema#boolean">{{ work.do_not_index }}</cdm:do_not_index>
1717
<cdm:work_date_document rdf:datatype="http://www.w3.org/2001/XMLSchema#date">{{ work.date_document }}</cdm:work_date_document>
1818
<cdm:work_created_by_agent rdf:resource="&cellar-authority;corporate-body/{{ work.created_by_agent }}"/>
19-
{% for lang in notice.languages %}
19+
{# <cdm:procurement_public_number_edition rdf:datatype="http://www.w3.org/2001/XMLSchema#positiveInteger">{{ notice.number_edition }}</cdm:procurement_public_number_edition> #}
20+
{% for lang in mets.languages %}
2021
<cdm:work_title xml:lang="{{ lang }}">{{ work.title[lang] }}</cdm:work_title>
2122
{% endfor %}
23+
{# <cdm:procurement_public_number_document_in_official-journal rdf:datatype="http://www.w3.org/2001/XMLSchema#string">{{ notice.number }}</cdm:procurement_public_number_document_in_official-journal> #}
2224
<cdm:datetime_transmission rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">{{ work.datetime_transmission }}</cdm:datetime_transmission>
2325
{# <cdm:procurement_public_issued_by_country>{{ work.procurement_public_issued_by_country }}</cdm:procurement_public_issued_by_country>
2426
{% for uri in work.procurement_public_url_etendering %}
@@ -36,16 +38,16 @@
3638
<cdm:work_dataset_has_frequency_publication_frequency rdf:resource="http://publications.europa.eu/resource/authority/frequency/{{ work.dataset_has_frequency_publication_frequency }}"/> #}
3739
</cdm:work>
3840

39-
<cdm:expression rdf:about="&resource;ted/{{ work.identifier }}.MUL">
41+
<cdm:expression rdf:about="&resource;ted/{{ expression.identifier }}">
4042
<cdm:expression_belongs_to_work rdf:resource="&resource;ted/{{ work.identifier }}"/>
41-
{% for lang in notice.languages %}
43+
{% for lang in mets.languages %}
4244
<cdm:expression_title xml:lang="{{ lang }}">{{ expression.title[lang] }}</cdm:expression_title>
4345
{% endfor %}
4446
<cdm:expression_uses_language rdf:resource="&cellar-authority;language/{{ expression.uses_language }}"/>
4547
</cdm:expression>
4648

47-
<cdm:manifestation_distribution rdf:about="&resource;ted/{{ work.identifier }}.MUL.rdf">
48-
<cdm:manifestation_manifests_expression rdf:resource="&resource;ted/{{ work.identifier }}.MUL"/>
49+
<cdm:manifestation_distribution rdf:about="&resource;ted/{{ manifestation.identifier }}">
50+
<cdm:manifestation_manifests_expression rdf:resource="&resource;ted/{{ expression.identifier }}"/>
4951
<cdm:manifestation_type rdf:datatype="http://www.w3.org/2001/XMLSchema#string">{{ manifestation.type }}</cdm:manifestation_type>
5052
<cdm:manifestation_date_publication rdf:datatype="http://www.w3.org/2001/XMLSchema#date">{{ manifestation.date_publication }}</cdm:manifestation_date_publication>
5153
{# <cdm:manifestation_distribution_has_status_distribution_status rdf:resource="http://publications.europa.eu/resource/authority/dataset-status/{{ manifestation.distribution_has_status_distribution_status }}"/>

‎ted_sws/notice_packager/resources/templates/tmd_rdf.jinja2‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
%cellarEntities;
66
]>
77
<rdf:RDF xmlns:tdm="http://publications.europa.eu/ontology/tdm#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
8-
<tdm:stream rdf:about="&resource;distribution/{{ notice.id }}/{{ notice.id }}.rdf">
9-
<tdm:stream_name rdf:datatype="http://www.w3.org/2001/XMLSchema#string">{{ notice.id }}.rdf</tdm:stream_name>
8+
<tdm:stream rdf:about="&resource;distribution/{{ notice.id }}/{{ mets.notice_file_href }}">
9+
<tdm:stream_name rdf:datatype="http://www.w3.org/2001/XMLSchema#string">{{ mets.notice_file_href }}</tdm:stream_name>
1010
<tdm:stream_format>
1111
<tdm:format>
1212
<tdm:format_designated_by_format_designation>
@@ -16,7 +16,7 @@
1616
</tdm:format_designated_by_format_designation>
1717
</tdm:format>
1818
</tdm:stream_format>
19-
<tdm:stream_label rdf:datatype="http://www.w3.org/2001/XMLSchema#string">{{ work.title[notice.languages[0]] }}</tdm:stream_label>
19+
<tdm:stream_label rdf:datatype="http://www.w3.org/2001/XMLSchema#string">{{ work.title[mets.languages[0]] }}</tdm:stream_label>
2020
<tdm:stream_order rdf:datatype="http://www.w3.org/2001/XMLSchema#positiveInteger">1</tdm:stream_order>
2121
</tdm:stream>
2222
</rdf:RDF>

‎ted_sws/notice_packager/services/metadata_transformer.py‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414
import datetime
1515

1616
from ted_sws.notice_metadata_processor.model.metadata import ExtractedMetadata
17-
from ted_sws.notice_packager.model.metadata import PackagerMetadata, ACTION_CREATE, LANGUAGE, REVISION, BASE_WORK, \
18-
BASE_TITLE
17+
from ted_sws.notice_packager.model.metadata import PackagerMetadata, METS_TYPE_CREATE, LANGUAGE, REVISION, BASE_WORK, \
18+
BASE_TITLE, METS_DMD_HREF, METS_DMD_ID, METS_TMD_ID, METS_TMD_HREF, METS_FILE_ID, METS_NOTICE_FILE_HREF
1919

2020
# This is used in pipeline
2121
NORMALIZED_SEPARATOR = '_'
@@ -31,9 +31,39 @@ class MetadataTransformer:
3131
def __init__(self, notice_metadata: ExtractedMetadata):
3232
self.notice_metadata = notice_metadata
3333

34-
def template_metadata(self, action: str = ACTION_CREATE) -> PackagerMetadata:
34+
def template_metadata(self, action: str = METS_TYPE_CREATE) -> PackagerMetadata:
3535
metadata = self.from_notice_metadata(self.notice_metadata)
36-
metadata.notice.action.type = action
36+
37+
# here are set custom and composed metadata properties
38+
metadata.mets.type = action
39+
metadata.mets.document_id = f"{metadata.work.identifier}_{action}"
40+
metadata.mets.dmd_href = METS_DMD_HREF.format(
41+
work_identifier=metadata.work.identifier,
42+
revision=metadata.mets.revision
43+
)
44+
metadata.mets.dmd_id = METS_DMD_ID.format(
45+
work_identifier=metadata.work.identifier,
46+
revision=metadata.mets.revision,
47+
dmd_idx="001"
48+
)
49+
metadata.mets.tmd_id = METS_TMD_ID.format(
50+
work_identifier=metadata.work.identifier,
51+
revision=metadata.mets.revision,
52+
tmd_idx="001"
53+
)
54+
metadata.mets.tmd_href = METS_TMD_HREF.format(
55+
work_identifier=metadata.work.identifier,
56+
revision=metadata.mets.revision
57+
)
58+
metadata.mets.file_id = METS_FILE_ID.format(
59+
work_identifier=metadata.work.identifier,
60+
revision=metadata.mets.revision,
61+
file_idx="001"
62+
)
63+
metadata.mets.notice_file_href = METS_NOTICE_FILE_HREF.format(
64+
work_identifier=metadata.work.identifier,
65+
revision=metadata.mets.revision
66+
)
3767
return metadata
3868

3969
@classmethod
@@ -46,16 +76,6 @@ def normalize_value(cls, value: str) -> str:
4676
"""
4777
return value.replace(DENORMALIZED_SEPARATOR, NORMALIZED_SEPARATOR)
4878

49-
@classmethod
50-
def denormalize_value(cls, value: str) -> str:
51-
"""
52-
The pipeline's separator is replaced with initial (TED API)'s one.
53-
This is used when notice goes out to API
54-
:param value:
55-
:return:
56-
"""
57-
return value.replace(NORMALIZED_SEPARATOR, DENORMALIZED_SEPARATOR)
58-
5979
@classmethod
6080
def from_notice_metadata(cls, notice_metadata: ExtractedMetadata) -> PackagerMetadata:
6181
_date = datetime.datetime.now()
@@ -65,6 +85,7 @@ def from_notice_metadata(cls, notice_metadata: ExtractedMetadata) -> PackagerMet
6585

6686
# NOTICE
6787
metadata.notice.id = cls.normalize_value(notice_metadata.notice_publication_number)
88+
metadata.notice.number = publication_notice_number(metadata.notice.id)
6889

6990
# WORK
7091
publication_date = datetime.datetime.strptime(notice_metadata.publication_date, '%Y%m%d').strftime('%Y-%m-%d')
@@ -81,9 +102,11 @@ def from_notice_metadata(cls, notice_metadata: ExtractedMetadata) -> PackagerMet
81102
metadata.work.procurement_public_url_etendering = notice_metadata.uri_list
82103

83104
# EXPRESSION
84-
metadata.expression.title = {LANGUAGE: BASE_TITLE + " " + metadata.notice.id}
105+
metadata.expression.identifier = f"{metadata.work.identifier}.MUL"
106+
metadata.expression.title = {LANGUAGE: BASE_TITLE + " " + metadata.work.identifier}
85107

86108
# MANIFESTATION
109+
metadata.manifestation.identifier = f"{metadata.expression.identifier}.rdf"
87110
metadata.manifestation.date_publication = publication_date
88111
return metadata
89112

0 commit comments

Comments
 (0)