feat: add cloud logging connector

Author: seolmin@megazone.com

pkg/pip_requirements.txt

@@ -1,3 +1,4 @@
 spaceone-core
 spaceone-api
+google-api-python-client
 schematics

src/cloudforet/monitoring/conf/proto_conf.py

@@ -1,4 +1,4 @@
 PROTO = {
-    'spaceone.monitoring.api.plugin.data_source': ['DataSource'],
-    'spaceone.monitoring.api.plugin.log': ['Log'],
+    'cloudforet.monitoring.api.plugin.data_source': ['DataSource'],
+    'cloudforet.monitoring.api.plugin.log': ['Log'],
 }

src/cloudforet/monitoring/connector/__init__.py

@@ -0,0 +1 @@
+from cloudforet.monitoring.connector.cloud_logging_connector import CloudLoggingConnector

src/cloudforet/monitoring/connector/cloud_logging_connector.py

@@ -1,10 +1,57 @@
 import logging
-from cloudforet.monitoring.libs.google_connector import GoogleConnector
+from cloudforet.monitoring.libs.google_cloud_connector import GoogleCloudConnector
 
 _LOGGER = logging.getLogger(__name__)
 
 
-class CloudTrailConnector(GoogleConnector):
+class CloudLoggingConnector(GoogleCloudConnector):
+    google_client_service = 'logging'
+    version = 'v2'
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+
+    def list_log_entries(self, params):
+        print(params)
+        query = params['query']
+        start = params['start']
+        end = params['end']
+
+        _query = {
+            'filter': self._generate_logging_filter(query, start, end),
+            'orderBy': 'timestamp desc'
+        }
+        print(_query)
+        request = self.client.entries().list(**_query)
+
+        logs = []
+        while request is not None:
+            response = request.execute()
+            logs = [log for log in response.get('entries', [])]
+            request = self.client.entries().list_next(previous_request=request, previous_response=response)
+        return logs
+
+    @staticmethod
+    def _generate_logging_filter(query, start, end):
+        logging_filter = ''
+        log_filters = query.get('filters', [])
+        for log_filter in log_filters:
+            _filter = []
+            resource_type = log_filter.get('resource_type')
+            labels = log_filter.get('labels', {})
+            if resource_type:
+                _filter.append(f'resource.type={resource_type}')
+            if labels:
+                for key, value in labels.items():
+                    _filter.append(f'{key}={value}')
+            if logging_filter:
+                logging_filter += f' OR ({" AND ".join(_filter)})'
+            else:
+                logging_filter += f'({" AND ".join(_filter)})'
+
+        if logging_filter:
+            logging_filter += f' AND timestamp >= "{start}" AND timestamp <= "{end}"'
+        else:
+            logging_filter += f'timestamp >= "{start}" AND timestamp <= "{end}"'
+
+        return logging_filter

src/cloudforet/monitoring/libs/google_cloud_connector.py

@@ -0,0 +1,51 @@
+import logging
+import google.oauth2.service_account
+import googleapiclient
+import googleapiclient.discovery
+import logging
+from spaceone.core.connector import BaseConnector
+
+DEFAULT_SCHEMA = 'google_oauth_client_id'
+_LOGGER = logging.getLogger(__name__)
+
+
+class GoogleCloudConnector(BaseConnector):
+    google_client_service = 'compute'
+    version = 'v1'
+
+    def __init__(self, **kwargs):
+        """
+        kwargs
+            - schema
+            - options
+            - secret_data
+
+        secret_data(dict)
+            - type: ..
+            - project_id: ...
+            - token_uri: ...
+            - ...
+        """
+
+        super().__init__(transaction=None, config=None)
+        secret_data = kwargs.get('secret_data')
+        self.project_id = secret_data.get('project_id')
+        self.credentials = google.oauth2.service_account.Credentials.from_service_account_info(secret_data)
+        self.client = googleapiclient.discovery.build(self.google_client_service,
+                                                      self.version,
+                                                      credentials=self.credentials)
+
+    def verify(self, **kwargs):
+        if self.client is None:
+            self.set_connect(**kwargs)
+
+    def generate_query(self, **query):
+        query.update({
+            'project': self.project_id,
+        })
+        return query
+
+    def list_zones(self, **query):
+        query = self.generate_query(**query)
+        result = self.client.zones().list(**query).execute()
+        return result.get('items', [])

src/cloudforet/monitoring/libs/google_connector.py

@@ -1,8 +1,8 @@
 import logging
 from spaceone.core.manager import BaseManager
-# from cloudforet.monitoring.connector.cloudtrail_connector import CloudTrailConnector
 from cloudforet.monitoring.model.data_source_response_model import DataSourceMetadata
 from cloudforet.monitoring.manager.metadata_manager import MetadataManager
+from cloudforet.monitoring.connector.cloud_logging_connector import CloudLoggingConnector
 from cloudforet.monitoring.conf.monitoring_conf import *
 
 _LOGGER = logging.getLogger(__name__)
@@ -21,6 +21,5 @@ def init(params):
         return response_model.to_primitive()
 
     def verify(self, params):
-        pass
-        # cloudtrail_connector: CloudTrailConnector = self.locator.get_connector('CloudTrailConnector', **params)
-        # cloudtrail_connector.set_client(DEFAULT_REGION)
+        cloud_logging_connector: CloudLoggingConnector = self.locator.get_connector(CloudLoggingConnector, **params)
+        cloud_logging_connector.verify()

src/cloudforet/monitoring/manager/data_source_manager.py

@@ -15,7 +15,7 @@ def __init__(self, **kwargs):
     @staticmethod
     def get_data_source_metadata():
         metadata = LogMetadata.set_fields(
-            name='cloudtrail-table',
+            name='cloud-logging-table',
             fields=[
                 # MoreField.data_source('Event Name', 'event_name', options={
                 #     'layout': {

src/cloudforet/monitoring/manager/metadata_manager.py

@@ -2,7 +2,15 @@
 import logging
 from spaceone.core.manager import BaseManager
 
+from cloudforet.monitoring.connector import CloudLoggingConnector
+
 
 class MonitoringManager(BaseManager):
     def __init__(self, transaction):
         super().__init__(transaction)
+
+    def list_logs(self, params):
+        cloud_logging_conn: CloudLoggingConnector = self.locator.get_connector(CloudLoggingConnector, **params)
+
+        logs = cloud_logging_conn.list_log_entries(params)
+        print(logs)

src/cloudforet/monitoring/manager/monitoring_manager.py

@@ -22,8 +22,8 @@ class MetaDataView(Model):
 
 class LogMetadata(Model):
     view = ModelType(MetaDataView)
-    required_keys = ListType(StringType, default=['data.cloudtrail'])
-    supported_providers = ListType(StringType, default=['aws'])
+    required_keys = ListType(StringType, default=['data.google_cloud_logging'])
+    supported_providers = ListType(StringType, default=['google_cloud'])
 
     @classmethod
     def set_fields(cls, name='', fields=[]):