Skip to content

Commit 0a2ec37

Browse files
Matheus-GrMatheus-Gr
committed
Refactoring client to use observer
1 parent cc64401 commit 0a2ec37

8 files changed

Lines changed: 98 additions & 203 deletions

File tree

‎learning_orchestra_client/builder/builder.py‎

Lines changed: 16 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
import requests
2-
import time
1+
from observer import Observer
32
from response_treat import ResponseTreat
43
from dataset.dataset import Dataset
4+
import requests
55

66

77
class Builder:
88
def __init__(self, ip_from_cluster):
9+
self.CLUSTER_IP = ip_from_cluster
910
self.cluster_url = "http://" + ip_from_cluster + \
1011
"/api/learningOrchestra/v1/builder"
1112
self.response_treat = ResponseTreat()
@@ -45,10 +46,10 @@ def run_builder_sync(self, train_dataset_name, test_dataset_name,
4546
+ test_dataset_name
4647
+ " ----------"
4748
)
48-
self.dataset.verify_dataset_processing_done(train_dataset_name,
49-
pretty_response)
50-
self.dataset.verify_dataset_processing_done(test_dataset_name,
51-
pretty_response)
49+
Observer(train_dataset_name, self.CLUSTER_IP).observe_processing(
50+
pretty_response)
51+
Observer(test_dataset_name, self.CLUSTER_IP).observe_processing(
52+
pretty_response)
5253
request_body_content = {
5354
self.TRAIN_NAME: train_dataset_name,
5455
self.TEST_NAME: test_dataset_name,
@@ -58,8 +59,8 @@ def run_builder_sync(self, train_dataset_name, test_dataset_name,
5859
response = requests.post(url=self.cluster_url,
5960
json=request_body_content)
6061
for i in model_classifier:
61-
self.verify_model_processing_done(test_dataset_name+i,
62-
pretty_response=pretty_response)
62+
Observer(test_dataset_name+i, self.CLUSTER_IP).observe_processing(
63+
pretty_response)
6364

6465
return self.response_treat.treatment(response, pretty_response)
6566

@@ -93,10 +94,10 @@ def run_builder_async(self, train_dataset_name, test_dataset_name,
9394
+ test_dataset_name
9495
+ " ----------"
9596
)
96-
self.dataset.verify_dataset_processing_done(train_dataset_name,
97-
pretty_response)
98-
self.dataset.verify_dataset_processing_done(test_dataset_name,
99-
pretty_response)
97+
Observer(train_dataset_name, self.CLUSTER_IP).observe_processing(
98+
pretty_response)
99+
Observer(test_dataset_name, self.CLUSTER_IP).observe_processing(
100+
pretty_response)
100101
request_body_content = {
101102
self.TRAIN_NAME: train_dataset_name,
102103
self.TEST_NAME: test_dataset_name,
@@ -140,9 +141,9 @@ def search_model_prediction(self, model_name, query={}, limit=10, skip=0,
140141
future content requests.
141142
"""
142143
cluster_url_dataset = self.cluster_url + "/" + model_name + \
143-
"?query=" + str(query) + \
144-
"&limit=" + str(limit) + \
145-
"&skip=" + str(skip)
144+
"?query=" + str(query) + \
145+
"&limit=" + str(limit) + \
146+
"&skip=" + str(skip)
146147
response = requests.get(cluster_url_dataset)
147148
return self.response_treat.treatment(response, pretty_response)
148149

@@ -179,24 +180,3 @@ def delete_model(self, model_name, pretty_response=False):
179180
cluster_url_dataset = self.cluster_url + "/" + model_name
180181
response = requests.delete(cluster_url_dataset)
181182
return self.response_treat.treatment(response, pretty_response)
182-
183-
def verify_model_processing_done(self, model_name,
184-
pretty_response=False):
185-
"""
186-
description: This method check from time to time using Time lib, if a
187-
model predictions has finished being inserted into the
188-
Learning Orchestra storage mechanism.
189-
190-
model_name: Represents the model predictions name.
191-
pretty_response: If true return indented string, else return dict.
192-
"""
193-
if pretty_response:
194-
print("\n---------- WAITING " + model_name + " FINISH ----------")
195-
while True:
196-
time.sleep(self.WAIT_TIME)
197-
response = self.search_model(model_name,
198-
pretty_response=False)
199-
if len(response["result"]) == 0:
200-
continue
201-
if response["result"][self.METADATA_INDEX]["finished"]:
202-
break

‎learning_orchestra_client/dataset/dataset.py‎

Lines changed: 16 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,19 @@
22
__credits__ = "all free source developers"
33
__status__ = "Prototype"
44

5+
from observer import Observer
56
from response_treat import ResponseTreat
67
import requests
7-
import time
88

99

1010
class Dataset:
1111
def __init__(self, ip_from_cluster):
1212
self.cluster_url = "http://" + ip_from_cluster + \
1313
"/api/learningOrchestra/v1/dataset"
14-
self.WAIT_TIME = 3
15-
self.METADATA_INDEX = 0
1614
self.response_treat = ResponseTreat()
15+
self.OUTPUT_NAME = "datasetName"
16+
self.URL = "datasetURI"
17+
self.CLUSTER_IP = ip_from_cluster
1718

1819
def insert_dataset_sync(self, dataset_name, url, pretty_response=False):
1920
"""
@@ -29,9 +30,10 @@ def insert_dataset_sync(self, dataset_name, url, pretty_response=False):
2930
indicating the correct operation.
3031
"""
3132
request_url = self.cluster_url
32-
request_body = {"datasetName": dataset_name, "url": url}
33+
request_body = {self.OUTPUT_NAME: dataset_name,
34+
self.URL: url}
3335
response = requests.post(url=request_url, json=request_body)
34-
self.verify_dataset_processing_done(dataset_name, pretty_response)
36+
Observer(dataset_name, self.CLUSTER_IP).observe_processing()
3537
if pretty_response:
3638
print("\n----------" + " CREATED FILE " + dataset_name + " -------"
3739
"---")
@@ -54,9 +56,12 @@ def insert_dataset_async(self, dataset_name, url, pretty_response=False):
5456
proceed future checks to verify if the dataset is inserted).
5557
"""
5658
request_url = self.cluster_url
57-
request_body = {"datasetName": dataset_name, "url": url}
59+
request_body = {self.OUTPUT_NAME: dataset_name,
60+
self.URL: url}
5861
response = requests.post(url=request_url, json=request_body)
59-
print("\n----------" + " CREATE FILE " + dataset_name + " ----------")
62+
if pretty_response:
63+
print("\n----------" + " CREATED FILE " + dataset_name + " -------"
64+
"---")
6065
return self.response_treat.treatment(response, pretty_response)
6166

6267
def search_all_datasets(self, pretty_response=False):
@@ -110,9 +115,9 @@ def search_dataset_content(self, dataset_name, query={}, limit=10, skip=0,
110115
"""
111116

112117
request_url = self.cluster_url + "/" + dataset_name + \
113-
"?query=" + str(query) + \
114-
"&limit=" + str(limit) + \
115-
"&skip=" + str(skip)
118+
"?query=" + str(query) + \
119+
"&limit=" + str(limit) + \
120+
"&skip=" + str(skip)
116121
response = requests.get(request_url)
117122
return self.response_treat.treatment(response, pretty_response)
118123

@@ -132,24 +137,4 @@ def delete_dataset(self, dataset_name, pretty_response=False):
132137
"""
133138
request_url = self.cluster_url + "/" + dataset_name
134139
response = requests.delete(request_url)
135-
return self.response_treat.treatment(response, pretty_response)
136-
137-
def verify_dataset_processing_done(self, dataset_name,
138-
pretty_response=False):
139-
"""
140-
description: This method check from time to time using Time lib, if a
141-
dataset has finished being inserted
142-
into the Learning Orchestra storage mechanism.
143-
144-
pretty_response: If true return indented string, else return dict.
145-
"""
146-
if pretty_response:
147-
print("\n---------- WAITING " + dataset_name + " FINISH ----------")
148-
while True:
149-
time.sleep(self.WAIT_TIME)
150-
response = self.search_dataset_content(dataset_name, limit=1,
151-
pretty_response=False)
152-
if len(response["result"]) == 0:
153-
continue
154-
if response["result"][self.METADATA_INDEX]["finished"]:
155-
break
140+
return self.response_treat.treatment(response, pretty_response)

‎learning_orchestra_client/explore/histogram.py‎

Lines changed: 10 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
1-
import time
2-
3-
import requests
4-
1+
from observer import Observer
52
from response_treat import ResponseTreat
63
from dataset.dataset import Dataset
4+
import requests
75

86

97
class Histogram:
108
def __init__(self, ip_from_cluster):
9+
self.CLUSTER_IP = ip_from_cluster
1110
self.cluster_url = "http://" + ip_from_cluster + \
1211
"/api/learningOrchestra/v1/explore/histogram"
13-
self.WAIT_TIME = 3
14-
self.METADATA_INDEX = 0
1512
self.response_treat = ResponseTreat()
1613
self.INPUT_NAME = "inputDatasetName"
1714
self.OUTPUT_NAME = "outputDatasetName"
@@ -38,11 +35,12 @@ def run_histogram_sync(self, dataset_name, histogram_name, fields,
3835
self.OUTPUT_NAME: histogram_name,
3936
self.FIELDS: fields,
4037
}
41-
self.dataset.verify_dataset_processing_done(dataset_name,
42-
pretty_response)
38+
Observer(dataset_name, self.CLUSTER_IP).observe_processing(
39+
pretty_response)
4340
request_url = self.cluster_url
4441
response = requests.post(url=request_url, json=request_body)
45-
self.verify_dataset_histogram_done(histogram_name, pretty_response)
42+
Observer(histogram_name, self.CLUSTER_IP).observe_processing(
43+
pretty_response)
4644
if pretty_response:
4745
print(
4846
"\n----------"
@@ -76,8 +74,8 @@ def run_histogram_async(self, dataset_name, histogram_name, fields,
7674
self.OUTPUT_NAME: histogram_name,
7775
self.FIELDS: fields,
7876
}
79-
self.dataset.verify_dataset_processing_done(dataset_name,
80-
pretty_response)
77+
Observer(dataset_name, self.CLUSTER_IP).observe_processing(
78+
pretty_response)
8179
request_url = self.cluster_url
8280
response = requests.post(url=request_url, json=request_body)
8381
if pretty_response:
@@ -130,28 +128,6 @@ def search_histogram_data(self, histogram_name, query={}, limit=10, skip=0,
130128
response = requests.get(cluster_url_histogram)
131129
return self.response_treat.treatment(response, pretty_response)
132130

133-
def verify_dataset_histogram_done(self, histogram_name,
134-
pretty_response=True):
135-
"""
136-
description: This method check from time to time using Time lib, if a
137-
histogram has finished being inserted into the Learning Orchestra
138-
storage mechanism.
139-
140-
histogram_name: Represents the histogram name.
141-
pretty_response: If true return indented string, else return dict.
142-
"""
143-
if pretty_response:
144-
print(
145-
"\n---------- WAITING " + histogram_name + " FINISH ----------")
146-
while True:
147-
time.sleep(self.WAIT_TIME)
148-
response = self.search_histogram_data(histogram_name, limit=1,
149-
pretty_response=False)
150-
if len(response["result"]) == 0:
151-
continue
152-
if response["result"][self.METADATA_INDEX]["finished"]:
153-
break
154-
155131
def delete_histogram(self, histogram_name, pretty_response=False):
156132
"""
157133
description: This method is responsible for deleting a histogram.
@@ -166,4 +142,4 @@ def delete_histogram(self, histogram_name, pretty_response=False):
166142
"""
167143
cluster_url_histogram = self.cluster_url + "/" + histogram_name
168144
response = requests.delete(cluster_url_histogram)
169-
return self.response_treat.treatment(response, pretty_response)
145+
return self.response_treat.treatment(response, pretty_response)

‎learning_orchestra_client/explore/pca.py‎

Lines changed: 19 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
import requests
21
import time
2+
3+
import requests
4+
5+
from observer import Observer
36
from response_treat import ResponseTreat
47
from dataset.dataset import Dataset
58
from PIL import Image
@@ -15,7 +18,7 @@ def __init__(self, ip_from_cluster):
1518
self.LABEL = "label"
1619
self.dataset = Dataset(ip_from_cluster)
1720
self.WAIT_TIME = 3
18-
self.METADATA_INDEX = 0
21+
self.CLUSTER_IP = ip_from_cluster
1922

2023
def run_pca_sync(self, dataset_name, pca_name, label,
2124
pretty_response=False):
@@ -39,12 +42,11 @@ def run_pca_sync(self, dataset_name, pca_name, label,
3942
self.OUTPUT_NAME: pca_name,
4043
self.LABEL: label,
4144
}
42-
self.dataset.verify_dataset_processing_done(dataset_name,
43-
pretty_response)
45+
Observer(dataset_name, self.CLUSTER_IP).observe_processing(
46+
pretty_response)
4447
request_url = self.cluster_url
4548
response = requests.post(url=request_url, json=request_body)
46-
self.verify_pca_processing_done(pca_name,
47-
pretty_response=pretty_response)
49+
self.verify_pca_exist(pca_name, pretty_response)
4850
if pretty_response:
4951
print(
5052
"\n----------"
@@ -80,8 +82,8 @@ def run_pca_async(self, dataset_name, pca_name, label,
8082
self.OUTPUT_NAME: pca_name,
8183
self.LABEL: label,
8284
}
83-
self.dataset.verify_dataset_processing_done(dataset_name,
84-
pretty_response)
85+
Observer(dataset_name, self.CLUSTER_IP).observe_processing(
86+
pretty_response)
8587
request_url = self.cluster_url
8688
response = requests.post(url=request_url, json=request_body)
8789
if pretty_response:
@@ -123,7 +125,7 @@ def search_pca_plot(self, pca_name, pretty_response=False):
123125
if pretty_response:
124126
print(
125127
"\n----------"
126-
+ " READ"
128+
+ " READ "
127129
+ pca_name
128130
+ " PCA IMAGE PLOT "
129131
+ " ----------"
@@ -149,7 +151,7 @@ def delete_pca_plot(self, pca_name, pretty_response=False):
149151
response = requests.delete(cluster_url_pca)
150152
return self.response_treat.treatment(response, pretty_response)
151153

152-
def verify_pca_exist(self, pca_name):
154+
def verify_pca_exist(self, pca_name, pretty_response=False):
153155
"""
154156
description: This method is responsible to verify if a PCA image
155157
exist into the Learning Orchestra storage mechanism.
@@ -158,27 +160,12 @@ def verify_pca_exist(self, pca_name):
158160
159161
return: True if the PCA requested exist, false if does not.
160162
"""
161-
all_pca = self.search_all_pca()
162-
pca_name += ".png"
163-
return pca_name in all_pca.get('result')
164-
165-
def verify_pca_processing_done(self, pca_name,
166-
pretty_response=False):
167-
"""
168-
description: This method check from time to time using Time lib, if a
169-
projection has finished being inserted into the Learning Orchestra
170-
storage mechanism.
171-
172-
pca_name: Name of PCA image plot.
173-
pretty_response: If true return indented string, else return dict.
174-
"""
175163
if pretty_response:
176-
print(
177-
"\n---------- WAITING " + pca_name + " FINISH -------"
178-
"---")
179-
while True:
164+
print("\n---------- CHECKING IF " + pca_name + " FINISHED "
165+
"----------")
166+
exist = False
167+
pca_name += ".png"
168+
while not exist:
180169
time.sleep(self.WAIT_TIME)
181-
response = self.verify_pca_exist(pca_name)
182-
if response:
183-
break
184-
continue
170+
all_pca = self.search_all_pca()
171+
exist = pca_name in all_pca.get('result')

0 commit comments

Comments
 (0)