SPARK_HOME=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_home"])')
MEMORY=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_worker_memory"])')
SPARK_MASTER=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_master"])')
CASSANDRA_HOST=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(",".join(config["cassandra_addresses"]))')
CASSANDRA_PORT=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["cassandra_port"])')
CASSANDRA_OUT_CONSISTENCY=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["cassandra_output_consistency"])')
EVENT_LOGGING=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_event_logging"])')

"$SPARK_HOME"/bin/spark-submit \
--master "$SPARK_MASTER" \
--conf spark.executor.memory="$MEMORY" \
--conf spark.cassandra.connection.host="$CASSANDRA_HOST" \
--conf spark.cassandra.connection.port="$CASSANDRA_PORT" \
--conf spark.cassandra.output.consistency.level="$CASSANDRA_OUT_CONSISTENCY" \
--conf spark.eventLog.enabled="$EVENT_LOGGING" \
--conf spark.sql.session.timeZone=UTC \
--conf spark.sql.extensions=com.datastax.spark.connector.CassandraSparkExtensions \
--packages com.datastax.spark:spark-cassandra-connector_2.12:3.2.0 \
./src/spark/main_partition.py