Skip to content

Commit c035fad

Browse files
Dragos0000Dragos0000
committed
updated mapping suite model and loader
1 parent 5273b4f commit c035fad

15 files changed

Lines changed: 114 additions & 49 deletions

File tree

‎ted_sws/core/model/transform.py‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import abc
1010
from datetime import datetime
1111
from enum import Enum
12-
from typing import List, Optional
12+
from typing import List, Optional, Union
1313

1414
from ted_sws.core.model import PropertyBaseModel
1515

@@ -36,11 +36,33 @@ class NoticeFileResource(FileResource):
3636
notice_id: str
3737

3838

39-
class MetadataConstraints(MappingSuiteComponent):
39+
40+
class MetadataConstraintsStandardForm(MappingSuiteComponent):
41+
"""
42+
Metadata constraints structure for Standard forms
43+
"""
44+
eforms_subtype: List[str]
45+
start_date: Optional[List[str]]
46+
end_date: Optional[List[str]]
47+
min_xsd_version: List[str]
48+
max_xsd_version: Optional[List[str]]
49+
50+
51+
class MetadataConstraintsEform(MappingSuiteComponent):
52+
"""
53+
Metadata constraints structure for eForms
4054
"""
55+
eforms_subtype: List[str]
56+
start_date: Optional[List[str]]
57+
end_date: Optional[List[str]]
58+
eforms_sdk_versions: List[str]
59+
4160

61+
class MetadataConstraints(MappingSuiteComponent):
62+
"""
63+
Metadata constraints
4264
"""
43-
constraints: dict
65+
constraints: Union[MetadataConstraintsStandardForm, MetadataConstraintsEform]
4466

4567

4668
class TransformationRuleSet(MappingSuiteComponent):

‎ted_sws/data_manager/adapters/mapping_suite_repository.py‎

Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99

1010
from ted_sws import config
1111
from ted_sws.core.model.transform import MappingSuite, FileResource, TransformationRuleSet, SHACLTestSuite, \
12-
SPARQLTestSuite, MetadataConstraints, TransformationTestData, ConceptualMapping
12+
SPARQLTestSuite, MetadataConstraints, TransformationTestData, ConceptualMapping, MappingSuiteType, \
13+
MetadataConstraintsStandardForm, MetadataConstraintsEform
1314
from ted_sws.data_manager.adapters import inject_date_string_fields, remove_date_string_fields
1415
from ted_sws.data_manager.adapters.repository_abc import MappingSuiteRepositoryABC
1516
from ted_sws.mapping_suite_processor.adapters.conceptual_mapping_reader import ConceptualMappingReader
@@ -25,8 +26,17 @@
2526
MS_CONCEPTUAL_MAPPING_FILE_NAME = "conceptual_mappings.xlsx"
2627
MS_OUTPUT_FOLDER_NAME = "output"
2728
MS_TEST_SUITE_REPORT = "test_suite_report"
28-
MS_CREATED_AT = "created_at"
29+
MS_CREATED_AT_KEY = "created_at"
2930
MONGODB_COLLECTION_ID = "_id"
31+
MS_METADATA_IDENTIFIER_KEY = 'identifier'
32+
MS_STANDARD_METADATA_VERSION_KEY = 'version'
33+
MS_EFORMS_METADATA_VERSION_KEY = 'mapping_version'
34+
MS_METADATA_CONSTRAINTS_KEY = 'metadata_constraints'
35+
MS_CONSTRAINTS_KEY = 'constraints'
36+
MS_TITLE_KEY = 'title'
37+
MS_HASH_DIGEST_KEY = 'mapping_suite_hash_digest'
38+
MS_MAPPING_TYPE_KEY = 'mapping_type'
39+
MS_ONTOLOGY_VERSION_KEY = 'ontology_version'
3040

3141

3242
class MappingSuiteRepositoryMongoDB(MappingSuiteRepositoryABC):
@@ -55,8 +65,8 @@ def _create_dict_from_mapping_suite(self, mapping_suite: MappingSuite) -> dict:
5565
"""
5666
mapping_suite_dict = mapping_suite.dict()
5767
mapping_suite_dict[MONGODB_COLLECTION_ID] = mapping_suite.get_mongodb_id()
58-
mapping_suite_dict[MS_CREATED_AT] = datetime.fromisoformat(mapping_suite_dict[MS_CREATED_AT])
59-
inject_date_string_fields(data=mapping_suite_dict, date_field_name=MS_CREATED_AT)
68+
mapping_suite_dict[MS_CREATED_AT_KEY] = datetime.fromisoformat(mapping_suite_dict[MS_CREATED_AT_KEY])
69+
inject_date_string_fields(data=mapping_suite_dict, date_field_name=MS_CREATED_AT_KEY)
6070
return mapping_suite_dict
6171

6272
def _create_mapping_suite_from_dict(self, mapping_suite_dict: dict) -> Optional[MappingSuite]:
@@ -67,8 +77,8 @@ def _create_mapping_suite_from_dict(self, mapping_suite_dict: dict) -> Optional[
6777
"""
6878
if mapping_suite_dict:
6979
mapping_suite_dict.pop(MONGODB_COLLECTION_ID, None)
70-
mapping_suite_dict[MS_CREATED_AT] = mapping_suite_dict[MS_CREATED_AT].isoformat()
71-
remove_date_string_fields(data=mapping_suite_dict, date_field_name=MS_CREATED_AT)
80+
mapping_suite_dict[MS_CREATED_AT_KEY] = mapping_suite_dict[MS_CREATED_AT_KEY].isoformat()
81+
remove_date_string_fields(data=mapping_suite_dict, date_field_name=MS_CREATED_AT_KEY)
7282
return MappingSuite(**mapping_suite_dict)
7383
return None
7484

@@ -134,7 +144,6 @@ def _read_package_metadata(self, package_path: pathlib.Path) -> dict:
134144
package_metadata_path = package_path / MS_METADATA_FILE_NAME
135145
package_metadata_content = package_metadata_path.read_text(encoding="utf-8")
136146
package_metadata = json.loads(package_metadata_content)
137-
package_metadata['metadata_constraints'] = MetadataConstraints(**package_metadata['metadata_constraints'])
138147
return package_metadata
139148

140149
def _read_transformation_rule_set(self, package_path: pathlib.Path) -> TransformationRuleSet:
@@ -343,16 +352,30 @@ def _read_mapping_suite_package(self, mapping_suite_identifier: str) -> Optional
343352
package_path = self.repository_path / mapping_suite_identifier
344353
if package_path.is_dir():
345354
package_metadata = self._read_package_metadata(package_path)
346-
package_metadata["version"] = package_metadata[
347-
"version"] if "version" in package_metadata else package_metadata["mapping_version"]
348-
package_metadata["identifier"] = package_metadata[
349-
"identifier"] if "identifier" in package_metadata else mapping_suite_identifier
350-
package_metadata["transformation_rule_set"] = self._read_transformation_rule_set(package_path)
351-
package_metadata["shacl_test_suites"] = self._read_shacl_test_suites(package_path)
352-
package_metadata["sparql_test_suites"] = self._read_sparql_test_suites(package_path)
353-
package_metadata["transformation_test_data"] = self._read_test_data_package(package_path)
354-
package_metadata["conceptual_mapping"] = self._read_conceptual_mapping(package_path)
355-
return MappingSuite(**package_metadata)
355+
if MS_MAPPING_TYPE_KEY in package_metadata and package_metadata[
356+
MS_MAPPING_TYPE_KEY] == MappingSuiteType.ELECTRONIC_FORMS:
357+
package_metadata[MS_METADATA_CONSTRAINTS_KEY] = MetadataConstraints(constraints=MetadataConstraintsEform(
358+
**package_metadata[MS_METADATA_CONSTRAINTS_KEY][MS_CONSTRAINTS_KEY]))
359+
else:
360+
package_metadata[MS_METADATA_CONSTRAINTS_KEY] = MetadataConstraints(constraints=MetadataConstraintsStandardForm(
361+
**package_metadata[MS_METADATA_CONSTRAINTS_KEY][MS_CONSTRAINTS_KEY]))
362+
mapping_suite = MappingSuite(metadata_constraints=package_metadata[MS_METADATA_CONSTRAINTS_KEY],
363+
created_at=package_metadata[MS_CREATED_AT_KEY],
364+
title=package_metadata[MS_TITLE_KEY],
365+
ontology_version=package_metadata[MS_ONTOLOGY_VERSION_KEY],
366+
mapping_suite_hash_digest=package_metadata[MS_HASH_DIGEST_KEY],
367+
mapping_type=package_metadata[MS_MAPPING_TYPE_KEY] if MS_MAPPING_TYPE_KEY in package_metadata else MappingSuiteType.STANDARD_FORMS,
368+
version=package_metadata[
369+
MS_STANDARD_METADATA_VERSION_KEY] if MS_STANDARD_METADATA_VERSION_KEY in package_metadata else \
370+
package_metadata[MS_EFORMS_METADATA_VERSION_KEY],
371+
identifier=package_metadata[
372+
MS_METADATA_IDENTIFIER_KEY] if MS_METADATA_IDENTIFIER_KEY in package_metadata else mapping_suite_identifier,
373+
transformation_rule_set=self._read_transformation_rule_set(package_path),
374+
shacl_test_suites=self._read_shacl_test_suites(package_path),
375+
sparql_test_suites=self._read_sparql_test_suites(package_path),
376+
transformation_test_data=self._read_test_data_package(package_path),
377+
conceptual_mapping=self._read_conceptual_mapping(package_path)) #TODO remove conceptual_mapping value assignment when conceptual mapping reader is removed
378+
return mapping_suite
356379
return None
357380

358381
@classmethod

‎ted_sws/mapping_suite_processor/services/mapping_suite_validation_service.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,11 @@
22
import pathlib
33
from typing import Optional
44

5-
from ted_sws.data_manager.adapters.mapping_suite_repository import MS_METADATA_FILE_NAME
5+
from ted_sws.data_manager.adapters.mapping_suite_repository import MS_METADATA_FILE_NAME, \
6+
MS_STANDARD_METADATA_VERSION_KEY, MS_METADATA_IDENTIFIER_KEY, \
7+
MS_EFORMS_METADATA_VERSION_KEY
68
from ted_sws.mapping_suite_processor.adapters.mapping_suite_structure_checker import MappingSuiteStructureValidator
79

8-
MAPPING_SUITE_METADATA_IDENTIFIER_KEY = 'identifier'
9-
STANDARD_MAPPING_SUITE_METADATA_VERSION_KEY = 'version'
10-
EFORMS_MAPPING_SUITE_METADATA_VERSION_KEY = 'mapping_version'
11-
1210

1311
def get_mapping_suite_id_from_file_system(mapping_suite_path: pathlib.Path) -> Optional[str]:
1412
"""
@@ -20,10 +18,10 @@ def get_mapping_suite_id_from_file_system(mapping_suite_path: pathlib.Path) -> O
2018

2119
if mapping_suite_metadata_path.exists() and mapping_suite_metadata_path.is_file():
2220
mapping_suite_metadata = json.loads(mapping_suite_metadata_path.read_text(encoding="utf-8"))
23-
identifier_value = mapping_suite_metadata[MAPPING_SUITE_METADATA_IDENTIFIER_KEY]
21+
identifier_value = mapping_suite_metadata[MS_METADATA_IDENTIFIER_KEY]
2422
version_value = mapping_suite_metadata[
25-
STANDARD_MAPPING_SUITE_METADATA_VERSION_KEY] if STANDARD_MAPPING_SUITE_METADATA_VERSION_KEY in mapping_suite_metadata else \
26-
mapping_suite_metadata[EFORMS_MAPPING_SUITE_METADATA_VERSION_KEY]
23+
MS_STANDARD_METADATA_VERSION_KEY] if MS_STANDARD_METADATA_VERSION_KEY in mapping_suite_metadata else \
24+
mapping_suite_metadata[MS_EFORMS_METADATA_VERSION_KEY]
2725
return f"{identifier_value}_v{version_value}"
2826
return None
2927

‎ted_sws/notice_metadata_processor/services/notice_eligibility.py‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,12 @@ def is_version_in_range(notice_metadata: NormalisedMetadata, mapping_suite: Mapp
4343
# we need to extract only the version i.e 1.7 or 1.7.9
4444
eforms_sdk_version = notice_xsd_version.rsplit('-', 1)[1]
4545
constraint_version_range = [format_version_with_zero_patch(version) for version in
46-
constraints[EFORMS_SDK_VERSIONS_KEY]]
46+
constraints.eforms_sdk_versions]
4747
return format_version_with_zero_patch(eforms_sdk_version) in constraint_version_range
4848
elif mapping_suite.mapping_type == MappingSuiteType.STANDARD_FORMS and notice_metadata.notice_source == NoticeSource.STANDARD_FORM:
4949
notice_xsd_version = notice_metadata.xsd_version
50-
constraint_min_xsd_version = constraints[MIN_XSD_VERSION_KEY][0]
51-
constraint_max_xsd_version = constraints[MAX_XSD_VERSION_KEY][0]
50+
constraint_min_xsd_version = constraints.min_xsd_version[0]
51+
constraint_max_xsd_version = constraints.max_xsd_version[0]
5252
return constraint_min_xsd_version <= notice_xsd_version <= constraint_max_xsd_version
5353

5454
return False
@@ -63,17 +63,15 @@ def check_package(mapping_suite: MappingSuite, notice_metadata: NormalisedMetada
6363
"""
6464

6565
constraints = mapping_suite.metadata_constraints.constraints
66-
6766
eform_subtype = notice_metadata.eforms_subtype
6867
notice_publication_date = datetime.datetime.fromisoformat(notice_metadata.publication_date)
6968
in_version_range = is_version_in_range(notice_metadata=notice_metadata, mapping_suite=mapping_suite)
70-
7169
in_date_range = is_date_in_range(publication_date=notice_publication_date,
72-
constraint_start_date_value=constraints[START_DATE_KEY],
73-
constraint_end_date_value=constraints[END_DATE_KEY])
70+
constraint_start_date_value=constraints.start_date,
71+
constraint_end_date_value=constraints.end_date)
7472
eform_subtype_constraint_values = [str(eforms_subtype_value) for eforms_subtype_value in
75-
constraints[E_FORMS_SUBTYPE_KEY]]
76-
covered_eform_type = eform_subtype in eform_subtype_constraint_values
73+
constraints.eforms_subtype]
74+
covered_eform_type = str(eform_subtype) in eform_subtype_constraint_values
7775

7876
return in_date_range and in_version_range and covered_eform_type
7977

‎tests/e2e/data_manager/conftest.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from ted_sws.core.model.manifestation import RDFManifestation
44
from ted_sws.core.model.notice import NoticeStatus
55
from ted_sws.core.model.transform import FileResource, SPARQLTestSuite, MetadataConstraints, TransformationRuleSet, \
6-
SHACLTestSuite, TransformationTestData, MappingSuite
6+
SHACLTestSuite, TransformationTestData, MappingSuite, MetadataConstraintsStandardForm
77
from ted_sws.data_manager.adapters.notice_repository import NoticeRepositoryInFileSystem
88
from tests import TEST_DATA_PATH
99

@@ -190,7 +190,7 @@ def sparql_test_suite_with_invalid_query(invalid_sparql_file):
190190

191191
@pytest.fixture
192192
def dummy_mapping_suite(sparql_test_suite, shacl_test_suite):
193-
metadata_constrains = MetadataConstraints(constraints=dict())
193+
metadata_constrains = MetadataConstraints(constraints=MetadataConstraintsStandardForm(eforms_subtype=[29],min_xsd_version=["R2.0.9.S01.E01"]))
194194
file_name = "fake_title.txt"
195195
empty_file_resource = FileResource(file_name=file_name, file_content="no content here", original_name=file_name)
196196
transformation_rule_set = TransformationRuleSet(resources=[empty_file_resource],

‎tests/test_data/notice_transformer/mapping_suite_processor_repository/test_package/metadata.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"created_at": "2022-03-20T19:46:14.069518",
44
"identifier": "test_package",
55
"version": "0.1",
6+
"mapping_suite_hash_digest": "1740b31999a492bbe80122f766f8d4c1dd7a3f720b95337a88a10cf673ceb223",
67
"ontology_version": "3.0.0.alpha",
78
"metadata_constraints": {
89
"constraints": {

‎tests/test_data/notice_transformer/test_repository/test_package/metadata.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"title": "no_title",
33
"created_at": "2022-03-20T19:46:14.069518",
44
"identifier": "test_package",
5+
"mapping_suite_hash_digest": "1740b31999a492bbe80122f766f8d4c1dd7a3f720b95337a88a10cf673ceb223",
56
"version": "1.0.1",
67
"ontology_version": "3.0.0.alpha",
78
"metadata_constraints": {

‎tests/test_data/notice_transformer/test_repository/test_package2/metadata.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"created_at": "2022-03-20T19:46:14.069518",
44
"identifier": "test_package2",
55
"version": "2.1.6",
6+
"mapping_suite_hash_digest": "1740b31999a492bbe80122f766f8d4c1dd7a3f720b95337a88a10cf673ceb223",
67
"ontology_version": "3.0.0.alpha",
78
"metadata_constraints": {
89
"constraints": {

‎tests/test_data/notice_transformer/test_repository/test_package3/metadata.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"identifier": "test_package3",
55
"version": "3.1.6",
66
"ontology_version": "3.0.0.alpha",
7+
"mapping_suite_hash_digest": "1740b31999a492bbe80122f766f8d4c1dd7a3f720b95337a88a10cf673ceb223",
78
"metadata_constraints": {
89
"constraints": {
910
"eforms_subtype": [

‎tests/test_data/notice_transformer/test_repository/test_package4/metadata.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"identifier": "package_EF16",
33
"title": "Package EF16 v1.2",
4+
"created_at": "2022-03-20T19:46:14.069518",
45
"description": "This is the conceptual mapping for bla bla bla",
56
"mapping_version": "3.0.0-alpha.1",
67
"ontology_version": "4.0.0",

0 commit comments

Comments
 (0)