| 12345678910111213141516171819 |
- SPARK_HOME=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_home"])')
- MEMORY=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_worker_memory"])')
- SPARK_MASTER=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_master"])')
- CASSANDRA_HOST=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(",".join(config["cassandra_addresses"]))')
- CASSANDRA_PORT=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["cassandra_port"])')
- CASSANDRA_OUT_CONSISTENCY=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["cassandra_output_consistency"])')
- EVENT_LOGGING=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_event_logging"])')
-
- "$SPARK_HOME"/bin/spark-submit \
- --master "$SPARK_MASTER" \
- --conf spark.executor.memory="$MEMORY" \
- --conf spark.cassandra.connection.host="$CASSANDRA_HOST" \
- --conf spark.cassandra.connection.port="$CASSANDRA_PORT" \
- --conf spark.cassandra.output.consistency.level="$CASSANDRA_OUT_CONSISTENCY" \
- --conf spark.eventLog.enabled="$EVENT_LOGGING" \
- --conf spark.sql.session.timeZone=UTC \
- --conf spark.sql.extensions=com.datastax.spark.connector.CassandraSparkExtensions \
- --packages com.datastax.spark:spark-cassandra-connector_2.12:3.2.0 \
- ./src/spark/main_partition.py
|
