refactoring of the triple store classes

Author: Mihai10Mihai

…sws/core/adapters/sparql_triple_store.py …data_manager/adapters/sparql_endpoint.pyted_sws/core/adapters/sparql_triple_store.py renamed to ted_sws/data_manager/adapters/sparql_endpoint.py ted_sws/core/adapters/sparql_triple_store.py renamed to ted_sws/data_manager/adapters/sparql_endpoint.py

@@ -15,7 +15,6 @@
 from string import Template
 
 DEFAULT_ENCODING = 'utf-8'
-DEFAULT_ENDPOINT_URL = "https://publications.europa.eu/webapi/rdf/sparql"
 
 
 class SubstitutionTemplate(Template):
@@ -37,14 +36,14 @@ def create_or_reuse_connection(endpoint_url: str):
         return SPARQLClientPool.connection_pool[endpoint_url]
 
 
-class TripleStoreABC(ABC):
+class TripleStoreEndpointABC(ABC):
     """
         This class provides an abstraction for a TripleStore.
     """
 
     @abstractmethod
     def with_query(self, sparql_query: str, substitution_variables: dict = None,
-                   sparql_prefixes: str = "") -> 'TripleStoreABC':
+                   sparql_prefixes: str = "") -> 'TripleStoreEndpointABC':
         """
             This method will take a query in a string format
         :param sparql_query:
@@ -55,7 +54,7 @@ def with_query(self, sparql_query: str, substitution_variables: dict = None,
 
     @abstractmethod
     def with_query_from_file(self, sparql_query_file_path: str, substitution_variables: dict = None,
-                             prefixes: str = "") -> 'TripleStoreABC':
+                             prefixes: str = "") -> 'TripleStoreEndpointABC':
         """
             This method will read a query from a file
         :param sparql_query_file_path:
@@ -79,13 +78,13 @@ def fetch_tree(self) -> dict:
         """
 
 
-class SPARQLTripleStore(TripleStoreABC):
+class SPARQLTripleStoreEndpoint(TripleStoreEndpointABC):
 
-    def __init__(self, endpoint_url: str = DEFAULT_ENDPOINT_URL):
+    def __init__(self, endpoint_url: str):
         self.endpoint = SPARQLClientPool.create_or_reuse_connection(endpoint_url)
 
     def with_query(self, sparql_query: str, substitution_variables: dict = None,
-                   sparql_prefixes: str = "") -> TripleStoreABC:
+                   sparql_prefixes: str = "") -> TripleStoreEndpointABC:
         """
             Set the query text and return the reference to self for chaining.
         :return:
@@ -100,7 +99,7 @@ def with_query(self, sparql_query: str, substitution_variables: dict = None,
         return self
 
     def with_query_from_file(self, sparql_query_file_path: str, substitution_variables: dict = None,
-                             prefixes: str = "") -> TripleStoreABC:
+                             prefixes: str = "") -> TripleStoreEndpointABC:
         """
             Set the query text and return the reference to self for chaining.
         :return:

…ocessor/adapters/allegro_triple_store.py …ws/data_manager/adapters/triple_store.pyted_sws/mapping_suite_processor/adapters/allegro_triple_store.py renamed to ted_sws/data_manager/adapters/triple_store.py ted_sws/mapping_suite_processor/adapters/allegro_triple_store.py renamed to ted_sws/data_manager/adapters/triple_store.py

@@ -3,8 +3,10 @@
 from franz.openrdf.repository import Repository
 from franz.openrdf.sail import AllegroGraphServer
 
+from ted_sws.data_manager.adapters.sparql_endpoint import TripleStoreEndpointABC, SPARQLTripleStoreEndpoint
 
-class VersatileTripleStoreABC:
+
+class TripleStoreABC:
     @abc.abstractmethod
     def add_data_to_repository(self, file_content: str, repository_name: str):
         """
@@ -23,20 +25,30 @@ def add_file_to_repository(self, file_path, repository_name):
         :return:
         """
 
+    def get_sparql_triple_store_endpoint(self, repository_name: str = None) -> TripleStoreEndpointABC:
+        """
+            Return a triple store SPARQL endpoint connection
+        """
 
-class AllegroGraphVersatileTripleStore(abc.ABC):
+
+class AllegroGraphTripleStore(abc.ABC):
     """
         This class is handling interactions with Allegro Graph triple store
         Note: If catalog name is not set, every operation will be executed at root level in the triple store
     """
 
-    def __init__(self, host: str, user: str, password: str, catalog_name=None):
+    def __init__(self, host: str, user: str, password: str, default_repository="default", catalog_name=None):
         self.host = host
         self.user = user
         self.password = password
         self.catalog_name = catalog_name
-        self.allegro = AllegroGraphServer(host=self.host, port=443,
-                                          user=self.user, password=self.password)
+        self.default_repository = default_repository
+        self.allegro = AllegroGraphServer(host=self.host,
+                                          port=443,
+                                          user=self.user,
+                                          password=self.password,
+                                          verifyhost=0,
+                                          verifypeer=0)
 
     def create_repository(self, repository_name: str):
         """
@@ -62,7 +74,8 @@ def _get_repository(self, repository_name: str) -> Repository:
         :param repository_name:
         :return:
         """
-        return self.allegro.openCatalog().getRepository(name=repository_name, access_verb=Repository.ACCESS)
+        name = repository_name if repository_name else self.default_repository
+        return self.allegro.openCatalog().getRepository(name=name, access_verb=Repository.ACCESS)
 
     def list_repositories(self):
         """
@@ -71,7 +84,7 @@ def list_repositories(self):
         """
         return self.allegro.openCatalog(name=self.catalog_name).listRepositories()
 
-    def add_data_to_repository(self, file_content: str, repository_name: str):
+    def add_data_to_repository(self, file_content: str, repository_name: str = None):
         """
         Method to add triples from a string
         :param file_content:
@@ -81,7 +94,7 @@ def add_data_to_repository(self, file_content: str, repository_name: str):
         repository = self._get_repository(repository_name=repository_name)
         repository.getConnection().addData(data=file_content)
 
-    def add_file_to_repository(self, file_path, repository_name):
+    def add_file_to_repository(self, file_path, repository_name: str = None):
         """
         Method to add triples from a file
         :param file_path:
@@ -90,3 +103,11 @@ def add_file_to_repository(self, file_path, repository_name):
         """
         repository = self._get_repository(repository_name=repository_name)
         repository.getConnection().addFile(filePath=file_path)
+
+    def get_sparql_triple_store_endpoint(self, repository_name: str = None) -> TripleStoreEndpointABC:
+        """
+            Return a triple store SPARQL endpoint connection
+        """
+        endpoint_url = f"{self.host}/repositories/{repository_name}"
+        sparql_endpoint = SPARQLTripleStoreEndpoint(endpoint_url=endpoint_url)
+        return sparql_endpoint

ted_sws/data_manager/entrypoints/cli/cmd_generate_mapping_resources.py

@@ -5,7 +5,7 @@
 import click
 
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH
-from ted_sws.core.adapters.sparql_triple_store import SPARQLTripleStore, TripleStoreABC
+from ted_sws.data_manager.adapters.sparql_endpoint import SPARQLTripleStoreEndpoint, TripleStoreEndpointABC
 from ted_sws.resources import QUERIES_PATH, MAPPING_FILES_PATH
 from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
 from ted_sws.data_manager.adapters.mapping_suite_repository import TRANSFORM_PACKAGE_NAME, RESOURCES_PACKAGE_NAME
@@ -29,7 +29,7 @@ def __init__(
             self,
             queries_folder,
             output_folder,
-            triple_store: TripleStoreABC = SPARQLTripleStore()
+            triple_store: TripleStoreEndpointABC = SPARQLTripleStoreEndpoint()
     ):
         super().__init__(name=CMD_NAME)
         self.queries_folder_path = Path(os.path.realpath(queries_folder))
@@ -59,7 +59,7 @@ def run(mapping_suite_id=None,
         opt_queries_folder: str = str(QUERIES_PATH),
         opt_output_folder: str = str(MAPPING_FILES_PATH),
         opt_mappings_folder: str = DEFAULT_MAPPINGS_PATH,
-        triple_store: TripleStoreABC = SPARQLTripleStore()):
+        triple_store: TripleStoreEndpointABC = SPARQLTripleStoreEndpoint()):
     """
     This method will generate a json file for each ran SPARQL query in the resources folder
     :param mapping_suite_id:
@@ -94,7 +94,7 @@ def run(mapping_suite_id=None,
               help="Use to overwrite default OUTPUT")
 @click.option('-m', '--opt-mappings-folder', default=DEFAULT_MAPPINGS_PATH)
 def main(mapping_suite_id, opt_queries_folder, opt_output_folder, opt_mappings_folder):
-    run(mapping_suite_id, opt_queries_folder, opt_output_folder, opt_mappings_folder, SPARQLTripleStore())
+    run(mapping_suite_id, opt_queries_folder, opt_output_folder, opt_mappings_folder, SPARQLTripleStoreEndpoint())
 
 
 if __name__ == '__main__':

ted_sws/mapping_suite_processor/services/load_mapping_suite_output_into_triple_store.py

@@ -1,10 +1,10 @@
 import pathlib
 
 from ted_sws import config
-from ted_sws.mapping_suite_processor.adapters.allegro_triple_store import AllegroGraphVersatileTripleStore
+from ted_sws.data_manager.adapters.triple_store import AllegroGraphTripleStore
 
 
-def repository_exists(triple_store: AllegroGraphVersatileTripleStore, repository_name) -> bool:
+def repository_exists(triple_store: AllegroGraphTripleStore, repository_name) -> bool:
     """
     Method to check if the repository is in the triple store
     :param triple_store:
@@ -35,8 +35,8 @@ def load_mapping_suite_output_into_triple_store(package_folder_path, allegro_hos
 
     ttl_files_paths = [str(path) for path in package_folder_path.glob("output/**/*.ttl")]
 
-    triple_store = AllegroGraphVersatileTripleStore(host=allegro_host, password=allegro_password,
-                                                    user=allegro_user, catalog_name=allegro_catalog_name)
+    triple_store = AllegroGraphTripleStore(host=allegro_host, password=allegro_password,
+                                           user=allegro_user, catalog_name=allegro_catalog_name)
 
     if repository_exists(triple_store=triple_store, repository_name=package_name):
         triple_store.delete_repository(repository_name=package_name)

ted_sws/notice_publisher_triple_store/services/load_transformed_notice_into_triple_store.py

@@ -2,13 +2,13 @@
     This module implements functionality to load a given notice into a triple store.
 """
 from ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC
-from ted_sws.mapping_suite_processor.adapters.allegro_triple_store import VersatileTripleStoreABC
+from ted_sws.data_manager.adapters.triple_store import TripleStoreEndpointABC
 
 DEFAULT_NOTICE_REPOSITORY_NAME = "notices"
 
 
 def load_notice_into_triple_store(notice_id: str, notice_repository: NoticeRepositoryABC,
-                                  triple_store: VersatileTripleStoreABC,
+                                  triple_store_repository: TripleStoreEndpointABC,
                                   repository_name: str = DEFAULT_NOTICE_REPOSITORY_NAME):
     """
 
@@ -18,4 +18,4 @@ def load_notice_into_triple_store(notice_id: str, notice_repository: NoticeRepos
         raise ValueError('Notice, with "%s" notice_id, was not found' % notice_id)
 
     rdf_manifestation_string = notice.rdf_manifestation.object_data
-    triple_store.add_data_to_repository(file_content=rdf_manifestation_string, repository_name=repository_name)
+    triple_store_repository.add_data_to_repository(file_content=rdf_manifestation_string, repository_name=repository_name)

tests/e2e/conftest.py

@@ -2,10 +2,30 @@
 from pymongo import MongoClient
 
 from ted_sws import config
+from ted_sws.data_manager.adapters.triple_store import AllegroGraphTripleStore
+from tests import TEST_DATA_PATH
 
 
 @pytest.fixture
 def mongodb_client():
     uri = config.MONGO_DB_AUTH_URL
     mongodb_client = MongoClient(uri)
     return mongodb_client
+
+
+@pytest.fixture
+def allegro_triple_store():
+    return AllegroGraphTripleStore(host=config.ALLEGRO_HOST, user=config.AGRAPH_SUPER_USER,
+                                   password=config.AGRAPH_SUPER_PASSWORD)
+
+
+@pytest.fixture
+def ttl_file():
+    path = TEST_DATA_PATH / "notice_transformer" / "test_repository" / "test_package" / "transformation" / "mappings" / "award_of_contract.rml.ttl"
+    return path.read_text()
+
+
+@pytest.fixture
+def path_ttl_file():
+    path = TEST_DATA_PATH / "notice_transformer" / "test_repository" / "test_package" / "transformation" / "mappings" / "complementary_information.rml.ttl"
+    return str(path)

tests/e2e/data_manager/conftest.py

@@ -6,7 +6,6 @@
     SHACLTestSuite, TransformationTestData, MappingSuite
 from tests import TEST_DATA_PATH
 
-
 @pytest.fixture
 def query_content():
     return """# title: Official name
@@ -239,3 +238,8 @@ def fake_mapping_suite_id() -> str:
 @pytest.fixture
 def invalid_mapping_suite_id() -> str:
     return "test_invalid_package"
+
+
+@pytest.fixture
+def cellar_sparql_endpoint():
+    return "https://publications.europa.eu/webapi/rdf/sparql"

…a_normaliser/test_sparql_triple_store.py …ger/test_sparql_triple_store_endpoint.pytests/e2e/metadata_normaliser/test_sparql_triple_store.py renamed to tests/e2e/data_manager/test_sparql_triple_store_endpoint.py tests/e2e/metadata_normaliser/test_sparql_triple_store.py renamed to tests/e2e/data_manager/test_sparql_triple_store_endpoint.py

@@ -1,11 +1,11 @@
 import pandas as pd
 import pytest
 
-from ted_sws.core.adapters.sparql_triple_store import SPARQLTripleStore
+from ted_sws.data_manager.adapters.sparql_endpoint import SPARQLTripleStoreEndpoint
 from tests import TEST_DATA_PATH
 
 
-def test_sparql_triple_store_with_query():
+def test_sparql_triple_store_with_query(cellar_sparql_endpoint):
     query = """prefix cdm: <http://publications.europa.eu/ontology/cdm#> 
 PREFIX dct: <http://purl.org/dc/terms/>
 PREFIX dc: <http://purl.org/dc/elements/1.1/>
@@ -24,7 +24,7 @@ def test_sparql_triple_store_with_query():
     # dc: identifier
 
     substitution_variables = {"value": 10}
-    execute_query = SPARQLTripleStore().with_query(
+    execute_query = SPARQLTripleStoreEndpoint(endpoint_url=cellar_sparql_endpoint).with_query(
         sparql_query=query, substitution_variables=substitution_variables)
 
     tabular_results = execute_query.fetch_tabular()
@@ -36,16 +36,16 @@ def test_sparql_triple_store_with_query():
                "value"] == "http://publications.europa.eu/resource/authority/buyer-legal-type/OP_DATPRO"
 
     with pytest.raises(Exception):
-        SPARQLTripleStore().with_query(sparql_query="").fetch_tree()
+        SPARQLTripleStoreEndpoint(endpoint_url=cellar_sparql_endpoint).with_query(sparql_query="").fetch_tree()
 
     with pytest.raises(Exception):
-        SPARQLTripleStore().with_query(sparql_query="").fetch_tabular()
+        SPARQLTripleStoreEndpoint(endpoint_url=cellar_sparql_endpoint).with_query(sparql_query="").fetch_tabular()
 
 
-def test_sparql_triple_store_with_query_from_file():
+def test_sparql_triple_store_with_query_from_file(cellar_sparql_endpoint):
     query_path = TEST_DATA_PATH / "sparql_queries" / "buyer_legal_type.rq"
     substitution_variables = {"value": 10}
-    execute_query = SPARQLTripleStore().with_query_from_file(sparql_query_file_path=query_path,substitution_variables=substitution_variables)
+    execute_query = SPARQLTripleStoreEndpoint(endpoint_url=cellar_sparql_endpoint).with_query_from_file(sparql_query_file_path=query_path, substitution_variables=substitution_variables)
 
     tabular_results = execute_query.fetch_tabular()
     tree_results = execute_query.fetch_tree()
@@ -56,6 +56,3 @@ def test_sparql_triple_store_with_query_from_file():
                "value"] == "http://publications.europa.eu/resource/authority/buyer-legal-type/OP_DATPRO"
 
     assert "europa.eu" in execute_query.__str__()
-
-
-

tests/e2e/data_manager/test_triple_store.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python3
+
+# minio_feature_store.py
+# Date:  21.07.2022
+# Author: Eugeniu Costetchi
+# Email: eugen@meaningfy.ws
+
+"""
+
+"""
+
+REPOSITORY_NAME = "this_is_a_test_repository"
+SPARQL_QUERY = "select * {?s ?p ?o} limit 10"
+
+
+def test_repository_creation(allegro_triple_store):
+    if REPOSITORY_NAME in allegro_triple_store.list_repositories():
+        allegro_triple_store.delete_repository(repository_name=REPOSITORY_NAME)
+
+    assert REPOSITORY_NAME not in allegro_triple_store.list_repositories()
+
+    allegro_triple_store.create_repository(repository_name=REPOSITORY_NAME)
+
+    assert REPOSITORY_NAME in allegro_triple_store.list_repositories()
+
+    allegro_triple_store.delete_repository(repository_name=REPOSITORY_NAME)
+
+
+def test_loading_data(allegro_triple_store, path_ttl_file):
+    if REPOSITORY_NAME not in allegro_triple_store.list_repositories():
+        allegro_triple_store.create_repository(repository_name=REPOSITORY_NAME)
+
+    allegro_triple_store.add_file_to_repository(file_path=path_ttl_file, repository_name=REPOSITORY_NAME)
+
+    sparql_endpoint = allegro_triple_store.get_sparql_triple_store_endpoint(repository_name=REPOSITORY_NAME)
+
+    assert sparql_endpoint
+
+    query_result = sparql_endpoint.with_query(sparql_query=SPARQL_QUERY).fetch_tree()
+
+    assert query_result
+    assert "head" in query_result
+    assert "results" in query_result
+    assert len(query_result["results"]["bindings"]) == 10