From 023d7c8b996a72aa880a89508d0294dee9c43685 Mon Sep 17 00:00:00 2001 From: nitowa Date: Mon, 29 Aug 2022 06:54:31 -0400 Subject: [PATCH] checkpoint dir to settings, rename main_back to main_with_collect --- settings.json | 1 + src/spark/main.py | 2 +- src/spark/main_graphs.py | 2 +- src/spark/{main_bak.py => main_with_collect.py} | 0 4 files changed, 3 insertions(+), 2 deletions(-) rename src/spark/{main_bak.py => main_with_collect.py} (100%) diff --git a/settings.json b/settings.json index dc8ba36..ce495d8 100644 --- a/settings.json +++ b/settings.json @@ -16,6 +16,7 @@ "spark_master": "spark://osboxes:7077", "spark_worker_memory": "1g", "spark_event_logging": "true", + "spark_checkpoint_dir": "./checkpoints", "debug": false } \ No newline at end of file diff --git a/src/spark/main.py b/src/spark/main.py index 939f3a0..13f0a35 100644 --- a/src/spark/main.py +++ b/src/spark/main.py @@ -56,7 +56,7 @@ class Master: master = Master(config) master.spark.catalog.clearCache() -master.spark.sparkContext.setCheckpointDir('./checkpoints') +master.spark.sparkContext.setCheckpointDir(config['spark_checkpoint_dir']) tx_df = master.get_tx_dataframe() #Turn transactions into a list of ('id', [addr, addr, ...]) diff --git a/src/spark/main_graphs.py b/src/spark/main_graphs.py index a4046c1..e4c31f0 100644 --- a/src/spark/main_graphs.py +++ b/src/spark/main_graphs.py @@ -45,7 +45,7 @@ class Master: # end class Master master = Master(config) -master.spark.sparkContext.setCheckpointDir('./checkpoints') # spark is really adamant it needs this even if the algorithm is set to the non-checkpointed version +master.spark.sparkContext.setCheckpointDir(config['spark_checkpoint_dir']) tx_df = master.get_tx_dataframe() diff --git a/src/spark/main_bak.py b/src/spark/main_with_collect.py similarity index 100% rename from src/spark/main_bak.py rename to src/spark/main_with_collect.py