diff --git a/benchmark/geo_runs.py b/benchmark/geo_runs.py index 7064864..16a5d01 100644 --- a/benchmark/geo_runs.py +++ b/benchmark/geo_runs.py @@ -425,10 +425,67 @@ def update_batch_size(json_file_path, mechanism_name, new_batch_size): except Exception as e: print(f"An error occurred: {e}") +# def update_parameters(json_file_path, mechanism_name, new_tx_size=None, new_rate=None, new_batch_size=None): +# try: +# with open(json_file_path, "r") as file: +# data = json.load(file) + +# if mechanism_name in data["remote"]: +# # Update tx_size +# if new_tx_size is not None: +# data["remote"][mechanism_name]["bench_params"]["tx_size"] = new_tx_size + +# # Update rate +# if new_rate is not None: +# if isinstance(new_rate, list): +# data["remote"][mechanism_name]["bench_params"]["rate"] = new_rate +# else: +# data["remote"][mechanism_name]["bench_params"]["rate"] = [new_rate] + +# # Update batch_size +# if new_batch_size is not None: +# if mechanism_name == "hotstuff": +# data["remote"][mechanism_name]["node_params"]["mempool"]["batch_size"] = new_batch_size +# elif mechanism_name == "bullshark" or mechanism_name == "cometbft": +# data["remote"][mechanism_name]["node_params"]["batch_size"] = new_batch_size + +# with open(json_file_path, "w") as file: +# json.dump(data, file, indent=4) +# print(f"Parameters for {mechanism_name} updated successfully.") +# else: +# print(f"Mechanism {mechanism_name} not found in the JSON file.") + +# except Exception as e: +# print(f"An error occurred: {e}") + +# def update_batch_size_relative(json_file_path, mechanism_name, relative_tx_rate): +# try: +# with open(json_file_path, "r") as file: +# data = json.load(file) + +# if mechanism_name in data["remote"]: +# if mechanism_name == "hotstuff": +# tx_rate = data["remote"][mechanism_name]["bench_params"]["rate"][0] +# tx_size = data["remote"][mechanism_name]["bench_params"]["tx_size"] +# data["remote"][mechanism_name]["node_params"]["mempool"]["batch_size"] = int(tx_size) * (int(tx_rate) + int(relative_tx_rate)) +# elif mechanism_name == "bullshark": +# data["remote"][mechanism_name]["node_params"]["batch_size"] = relative_tx_rate +# with open(json_file_path, "w") as file: +# json.dump(data, file, indent=4) +# print(f"Batch count for {mechanism_name} updated to {tx_rate + relative_tx_rate}.") +# else: +# print(f"Mechanism {mechanism_name} not found in the JSON file.") + +# except Exception as e: +# print(f"An error occurred: {e}") + if __name__ == "__main__": - batch_sizes = [512, 1024, 10000, 20000, 50000, 80000, 100000, 200000, 300000] + # batch_sizes = [512, 1024, 10000, 20000, 50000, 80000, 100000, 200000, 300000] + batch_sizes = [1000, 5000, 10000, 100000, 1000000, 5000000, 7500000, 10000000, 12500000, 15000000] + # batch_sizes = [-100, 100, -10, 0, 10, 20, 30, 40, 50, 70] + mechanism = ["hotstuff"] print("Starting benchmarking tool") diff --git a/benchmark/remote.py b/benchmark/remote.py index da33fa6..044b488 100644 --- a/benchmark/remote.py +++ b/benchmark/remote.py @@ -249,11 +249,16 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals # for the faulty nodes to be online). committee = Committee.load(PathMaker.committee_file()) addresses = [f'{x}:{self.settings.ports["front"]}' for x in hosts] - rate_share = ceil(rate / committee.size()) # Take faults into account. + rate_share = ceil(rate / (committee.size() * 2)) # Take faults into account. timeout = node_parameters.timeout_delay - client_logs = [PathMaker.client_log_file(i) for i in range(len(hosts))] - for host, addr, log_file in zip(hosts, addresses, client_logs): - cmd = CommandMaker.run_client( + + # Updated client logs for two clients per node + client_logs_a = [PathMaker.client_log_file(f"{i}-a") for i in range(len(hosts))] + client_logs_b = [PathMaker.client_log_file(f"{i}-b") for i in range(len(hosts))] + + for host, addr, log_file_a, log_file_b in zip(hosts, addresses, client_logs_a, client_logs_b): + # Run first client (client-{i}-a) + cmd_a = CommandMaker.run_client( addr, bench_parameters.tx_size, rate_share, @@ -261,7 +266,18 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals timeout, nodes=addresses, ) - self._background_run(host, cmd, log_file) + self._background_run(host, cmd_a, log_file_a) + + # Run second client (client-{i}-b) + cmd_b = CommandMaker.run_client( + addr, + bench_parameters.tx_size, + rate_share, + self.mechanism.name, + timeout, + nodes=addresses, + ) + self._background_run(host, cmd_b, log_file_b) # Run the nodes. key_files = [PathMaker.key_file(i) for i in range(len(hosts))] @@ -411,7 +427,8 @@ def _logs(self, hosts, faults, committee=[]): # , servers, run_id): for i, host in enumerate(progress): c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect) c.get(PathMaker.node_log_file(i), local=PathMaker.node_log_file(i)) - c.get(PathMaker.client_log_file(i), local=PathMaker.client_log_file(i)) + c.get(PathMaker.client_log_file(f"{i}-a"), local=PathMaker.client_log_file(f"{i}-a")) + c.get(PathMaker.client_log_file(f"{i}-b"), local=PathMaker.client_log_file(f"{i}-b")) if self.mechanism.name == "cometbft": c.get( PathMaker.latency_log_file(i), diff --git a/benchmark/utils.py b/benchmark/utils.py index 1d44770..25dfaee 100644 --- a/benchmark/utils.py +++ b/benchmark/utils.py @@ -65,7 +65,7 @@ def node_log_file(i): @staticmethod def client_log_file(i): - assert isinstance(i, int) and i >= 0 + # assert isinstance(i, int) and i >= 0 return join(PathMaker.logs_path(), f'client-{i}.log') @staticmethod diff --git a/fab-params.json b/fab-params.json index 49033dc..60b3ab1 100644 --- a/fab-params.json +++ b/fab-params.json @@ -7,7 +7,7 @@ 4 ], "rate": [ - 200000 + 160000 ], "tx_size": 128, "duration": 300, @@ -22,8 +22,8 @@ "gc_depth": 50, "sync_retry_delay": 5000, "sync_retry_nodes": 3, - "batch_size": 300000, - "max_batch_delay": 100 + "batch_size": 15000000, + "max_batch_delay": 5000 } } }, diff --git a/results/metrics.csv b/results/metrics.csv index 48b8af6..8acd34d 100644 --- a/results/metrics.csv +++ b/results/metrics.csv @@ -683,3 +683,85 @@ run_id,name,faults,input_rate,committee_size,transaction_size,execution_time,bat 682,hotstuff,0.0,80000.0,4.0,128.0,3.0,15000000.0,1527777.0,195555483.0,36.0,18556.0,2375169.0,1608.0, 683,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0, 684,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,509259.0,65185161.0,12.0,6185.333333333333,791723.0,536.0,3.0 +685,hotstuff,0.0,80000.0,4.0,128.0,299.0,7500000.0,80409.0,10292323.0,221.0,79807.0,10215267.0,2166.0, +686,hotstuff,0.0,160000.0,4.0,128.0,230.0,7500000.0,151399.0,19379072.0,1153.0,150213.0,19227306.0,3779.0, +687,hotstuff,0.0,160000.0,4.0,128.0,175.0,7500000.0,94760.0,12129221.0,21087.0,93641.0,11986029.0,28540.0, +688,hotstuff,0.0,160000.0,4.0,128.0,12.0,7500000.0,174740.0,22366695.0,396.0,141324.0,18089491.0,2960.0, +689,hotstuff,0.0,160000.0,4.0,128.0,230.0,7500000.0,140299.66666666666,17958329.333333332,7545.333333333333,128392.66666666669,16434275.333333334,11759.666666666666,3.0 +690,hotstuff,0.0,80000.0,4.0,128.0,126.0,1000.0,39465.0,5051479.0,5675.0,39461.0,5051038.0,8319.0, +691,hotstuff,0.0,80000.0,4.0,128.0,181.0,1000.0,69051.0,8838542.0,2714.0,69045.0,8837761.0,4227.0, +692,hotstuff,0.0,80000.0,4.0,128.0,90.0,1000.0,30433.0,3895426.0,7108.0,30429.0,3894907.0,9650.0, +693,hotstuff,0.0,80000.0,4.0,128.0,126.0,1000.0,46316.333333333336,5928482.333333333,5165.666666666667,46311.66666666666,5927902.0,7398.666666666667,3.0 +694,hotstuff,0.0,80000.0,4.0,128.0,301.0,5000.0,80007.0,10240851.0,41.0,80007.0,10240885.0,180.0, +695,hotstuff,0.0,80000.0,4.0,128.0,301.0,5000.0,79998.0,10239710.0,50.0,79996.0,10239540.0,177.0, +696,hotstuff,0.0,80000.0,4.0,128.0,299.0,5000.0,79988.0,10238512.0,28.0,79985.0,10238136.0,162.0, +697,hotstuff,0.0,80000.0,4.0,128.0,301.0,5000.0,79997.66666666667,10239691.0,39.66666666666666,79996.0,10239520.333333334,173.0,3.0 +698,hotstuff,0.0,80000.0,4.0,128.0,301.0,10000.0,80057.0,10247240.0,19.0,80006.0,10240804.0,185.0, +699,hotstuff,0.0,80000.0,4.0,128.0,300.0,10000.0,79995.0,10239391.0,11.0,79993.0,10239153.0,75.0, +700,hotstuff,0.0,80000.0,4.0,128.0,300.0,10000.0,80010.0,10241257.0,13.0,80008.0,10241052.0,126.0, +701,hotstuff,0.0,80000.0,4.0,128.0,301.0,10000.0,80020.66666666667,10242629.333333334,14.333333333333334,80002.33333333333,10240336.333333334,128.66666666666666,3.0 +702,hotstuff,0.0,80000.0,4.0,128.0,301.0,100000.0,80002.0,10240314.0,7.0,79999.0,10239871.0,93.0, +703,hotstuff,0.0,80000.0,4.0,128.0,301.0,100000.0,80008.0,10241087.0,6.0,80005.0,10240610.0,76.0, +704,hotstuff,0.0,80000.0,4.0,128.0,301.0,100000.0,80004.0,10240532.0,10.0,80001.0,10240157.0,133.0, +705,hotstuff,0.0,80000.0,4.0,128.0,301.0,100000.0,80004.66666666667,10240644.333333334,7.666666666666667,80001.66666666667,10240212.666666666,100.66666666666669,3.0 +706,hotstuff,0.0,80000.0,4.0,128.0,300.0,1000000.0,80086.0,10250983.0,11.0,79983.0,10237814.0,268.0, +707,hotstuff,0.0,80000.0,4.0,128.0,300.0,1000000.0,80080.0,10250265.0,15.0,79981.0,10237575.0,367.0, +708,hotstuff,0.0,80000.0,4.0,128.0,300.0,1000000.0,80088.0,10251256.0,17.0,79985.0,10238120.0,425.0, +709,hotstuff,0.0,80000.0,4.0,128.0,300.0,1000000.0,80084.66666666667,10250834.666666666,14.333333333333334,79983.0,10237836.333333334,353.3333333333333,3.0 +710,hotstuff,0.0,80000.0,4.0,128.0,298.0,5000000.0,80397.0,10290810.0,51.0,79813.0,10216111.0,1265.0, +711,hotstuff,0.0,80000.0,4.0,128.0,299.0,5000000.0,80503.0,10304377.0,154.0,79937.0,10231986.0,1729.0, +712,hotstuff,0.0,80000.0,4.0,128.0,299.0,5000000.0,80511.0,10305418.0,68.0,79953.0,10233937.0,1335.0, +713,hotstuff,0.0,80000.0,4.0,128.0,298.0,5000000.0,80470.33333333333,10300201.666666666,91.0,79901.0,10227344.666666666,1443.0,3.0 +714,hotstuff,0.0,80000.0,4.0,128.0,299.0,7500000.0,80739.0,10334530.0,121.0,79903.0,10227556.0,2254.0, +715,hotstuff,0.0,80000.0,4.0,128.0,300.0,7500000.0,80401.0,10291293.0,97.0,79561.0,10183851.0,2087.0, +716,hotstuff,0.0,80000.0,4.0,128.0,299.0,7500000.0,80731.0,10333588.0,170.0,79896.0,10226633.0,2446.0, +717,hotstuff,0.0,80000.0,4.0,128.0,299.0,7500000.0,80623.66666666667,10319803.666666666,129.33333333333334,79786.66666666667,10212680.0,2262.333333333333,3.0 +718,hotstuff,0.0,80000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +719,hotstuff,0.0,80000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +720,hotstuff,0.0,80000.0,4.0,128.0,3.0,10000000.0,2307690.0,295384355.0,26.0,18541.0,2373300.0,1760.0, +721,hotstuff,0.0,80000.0,4.0,128.0,0.0,10000000.0,769230.0,98461451.66666669,8.666666666666666,6180.333333333333,791100.0,586.6666666666666,3.0 +722,hotstuff,0.0,80000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0, +723,hotstuff,0.0,80000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0, +724,hotstuff,0.0,80000.0,4.0,128.0,3.0,12500000.0,1999997.0,255999613.0,29.0,18471.0,2364331.0,1702.0, +725,hotstuff,0.0,80000.0,4.0,128.0,0.0,12500000.0,666665.6666666666,85333204.33333333,9.666666666666666,6157.0,788110.3333333334,567.3333333333334,3.0 +726,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +727,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +728,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +729,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0 +730,hotstuff,0.0,160000.0,4.0,128.0,20.0,1000.0,108427.0,13878697.0,1156.0,108358.0,13869837.0,1361.0, +731,hotstuff,0.0,160000.0,4.0,128.0,25.0,1000.0,79871.0,10223544.0,4682.0,79814.0,10216230.0,5551.0, +732,hotstuff,0.0,160000.0,4.0,128.0,57.0,1000.0,28409.0,3636357.0,9338.0,28406.0,3635910.0,9966.0, +733,hotstuff,0.0,160000.0,4.0,128.0,20.0,1000.0,72235.66666666667,9246199.333333334,5058.666666666667,72192.66666666667,9240659.0,5626.0,3.0 +734,hotstuff,0.0,160000.0,4.0,128.0,300.0,5000.0,159981.0,20477529.0,28.0,159978.0,20477120.0,94.0, +735,hotstuff,0.0,160000.0,4.0,128.0,227.0,5000.0,118457.0,15162517.0,6577.0,118454.0,15162116.0,10296.0, +736,hotstuff,0.0,160000.0,4.0,128.0,300.0,5000.0,128076.0,16393719.0,2893.0,128074.0,16393445.0,4505.0, +737,hotstuff,0.0,160000.0,4.0,128.0,300.0,5000.0,135504.66666666666,17344588.333333332,3166.0,135502.0,17344227.0,4965.0,3.0 +738,hotstuff,0.0,160000.0,4.0,128.0,300.0,10000.0,159963.0,20475215.0,246.0,159920.0,20469751.0,985.0, +739,hotstuff,0.0,160000.0,4.0,128.0,301.0,100000.0,160010.0,20481312.0,20.0,160004.0,20480562.0,180.0, +740,hotstuff,0.0,160000.0,4.0,128.0,300.0,100000.0,159976.0,20476964.0,38.0,159970.0,20476145.0,265.0, +741,hotstuff,0.0,160000.0,4.0,128.0,300.0,100000.0,160006.0,20480792.0,8.0,159999.0,20479836.0,37.0, +742,hotstuff,0.0,160000.0,4.0,128.0,301.0,100000.0,159997.33333333334,20479689.33333333,22.0,159991.0,20478847.666666668,160.66666666666666,3.0 +743,hotstuff,0.0,160000.0,4.0,128.0,300.0,1000000.0,160039.0,20485029.0,26.0,159941.0,20472477.0,333.0, +744,hotstuff,0.0,160000.0,4.0,128.0,300.0,1000000.0,160076.0,20489712.0,22.0,159969.0,20475987.0,276.0, +745,hotstuff,0.0,160000.0,4.0,128.0,300.0,1000000.0,160081.0,20490395.0,16.0,159975.0,20476738.0,184.0, +746,hotstuff,0.0,160000.0,4.0,128.0,300.0,1000000.0,160065.33333333334,20488378.666666668,21.33333333333333,159961.66666666666,20475067.33333333,264.3333333333333,3.0 +747,hotstuff,0.0,160000.0,4.0,128.0,299.0,5000000.0,158592.0,20299779.0,154.0,157999.0,20223857.0,1145.0, +748,hotstuff,0.0,160000.0,4.0,128.0,300.0,5000000.0,160359.0,20525906.0,183.0,159798.0,20454111.0,1128.0, +749,hotstuff,0.0,160000.0,4.0,128.0,291.0,5000000.0,159055.0,20359059.0,353.0,158450.0,20281655.0,1868.0, +750,hotstuff,0.0,160000.0,4.0,128.0,299.0,5000000.0,159335.33333333334,20394914.666666668,230.0,158749.0,20319874.33333333,1380.3333333333333,3.0 +751,hotstuff,0.0,160000.0,4.0,128.0,300.0,7500000.0,157147.0,20114816.0,502.0,156269.0,20002489.0,2531.0, +752,hotstuff,0.0,160000.0,4.0,128.0,299.0,7500000.0,148600.0,19020780.0,663.0,147781.0,18916002.0,2529.0, +753,hotstuff,0.0,160000.0,4.0,128.0,301.0,7500000.0,160084.0,20490794.0,221.0,159176.0,20374491.0,1515.0, +754,hotstuff,0.0,160000.0,4.0,128.0,300.0,7500000.0,155277.0,19875463.33333333,462.0,154408.66666666666,19764327.33333333,2191.6666666666665,3.0 +755,hotstuff,0.0,160000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +756,hotstuff,0.0,160000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +757,hotstuff,0.0,160000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +758,hotstuff,0.0,160000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0 +759,hotstuff,0.0,160000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0, +760,hotstuff,0.0,160000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0, +761,hotstuff,0.0,160000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0, +762,hotstuff,0.0,160000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0 +763,hotstuff,0.0,160000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +764,hotstuff,0.0,160000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +765,hotstuff,0.0,160000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0, +766,hotstuff,0.0,160000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0