checkpoint dir to settings, rename main_back to main_with_collect

settings.json

@@ -16,6 +16,7 @@
     "spark_master": "spark://osboxes:7077",
     "spark_worker_memory": "1g",
     "spark_event_logging": "true",
+    "spark_checkpoint_dir": "./checkpoints",
 
     "debug": false
 }

src/spark/main.py

@@ -56,7 +56,7 @@ class Master:
 
 master = Master(config)
 master.spark.catalog.clearCache()
-master.spark.sparkContext.setCheckpointDir('./checkpoints')
+master.spark.sparkContext.setCheckpointDir(config['spark_checkpoint_dir'])
 tx_df = master.get_tx_dataframe()
 
 #Turn transactions into a list of ('id', [addr, addr, ...])

src/spark/main_graphs.py

@@ -45,7 +45,7 @@ class Master:
 # end class Master
 
 master = Master(config)
-master.spark.sparkContext.setCheckpointDir('./checkpoints')  # spark is really adamant it needs this even if the algorithm is set to the non-checkpointed version
+master.spark.sparkContext.setCheckpointDir(config['spark_checkpoint_dir'])
 
 tx_df = master.get_tx_dataframe()