diff --git a/settings.json b/settings.json index dc8ba36..ce495d8 100644 --- a/settings.json +++ b/settings.json @@ -16,6 +16,7 @@ "spark_master": "spark://osboxes:7077", "spark_worker_memory": "1g", "spark_event_logging": "true", + "spark_checkpoint_dir": "./checkpoints", "debug": false } \ No newline at end of file diff --git a/src/spark/main.py b/src/spark/main.py index 939f3a0..13f0a35 100644 --- a/src/spark/main.py +++ b/src/spark/main.py @@ -56,7 +56,7 @@ class Master: master = Master(config) master.spark.catalog.clearCache() -master.spark.sparkContext.setCheckpointDir('./checkpoints') +master.spark.sparkContext.setCheckpointDir(config['spark_checkpoint_dir']) tx_df = master.get_tx_dataframe() #Turn transactions into a list of ('id', [addr, addr, ...]) diff --git a/src/spark/main_graphs.py b/src/spark/main_graphs.py index a4046c1..e4c31f0 100644 --- a/src/spark/main_graphs.py +++ b/src/spark/main_graphs.py @@ -45,7 +45,7 @@ class Master: # end class Master master = Master(config) -master.spark.sparkContext.setCheckpointDir('./checkpoints') # spark is really adamant it needs this even if the algorithm is set to the non-checkpointed version +master.spark.sparkContext.setCheckpointDir(config['spark_checkpoint_dir']) tx_df = master.get_tx_dataframe() diff --git a/src/spark/main_bak.py b/src/spark/main_with_collect.py similarity index 100% rename from src/spark/main_bak.py rename to src/spark/main_with_collect.py