SPARK_HOME=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_home"])') MEMORY=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_worker_memory"])') SPARK_MASTER=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_master"])') CASSANDRA_HOST=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(",".join(config["cassandra_addresses"]))') CASSANDRA_PORT=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["cassandra_port"])') CASSANDRA_OUT_CONSISTENCY=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["cassandra_output_consistency"])') EVENT_LOGGING=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_event_logging"])') "$SPARK_HOME"/bin/spark-submit \ --master "$SPARK_MASTER" \ --conf spark.executor.memory="$MEMORY" \ --conf spark.cassandra.connection.host="$CASSANDRA_HOST" \ --conf spark.cassandra.connection.port="$CASSANDRA_PORT" \ --conf spark.cassandra.output.consistency.level="$CASSANDRA_OUT_CONSISTENCY" \ --conf spark.eventLog.enabled="$EVENT_LOGGING" \ --conf spark.sql.session.timeZone=UTC \ --conf spark.sql.extensions=com.datastax.spark.connector.CassandraSparkExtensions \ --packages com.datastax.spark:spark-cassandra-connector_2.12:3.2.0 \ --jars ./spark-packages/graphframe_3.3.jar \ ./src/spark/main_graphs.py