You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

12345678910111213141516171819
  1. SPARK_HOME=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_home"])')
  2. MEMORY=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_worker_memory"])')
  3. SPARK_MASTER=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_master"])')
  4. CASSANDRA_HOST=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(",".join(config["cassandra_addresses"]))')
  5. CASSANDRA_PORT=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["cassandra_port"])')
  6. CASSANDRA_OUT_CONSISTENCY=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["cassandra_output_consistency"])')
  7. EVENT_LOGGING=$(python3 -c 'import json,sys;config=json.load(open("./settings.json"));print(config["spark_event_logging"])')
  8. "$SPARK_HOME"/bin/spark-submit \
  9. --master "$SPARK_MASTER" \
  10. --conf spark.executor.memory="$MEMORY" \
  11. --conf spark.cassandra.connection.host="$CASSANDRA_HOST" \
  12. --conf spark.cassandra.connection.port="$CASSANDRA_PORT" \
  13. --conf spark.cassandra.output.consistency.level="$CASSANDRA_OUT_CONSISTENCY" \
  14. --conf spark.eventLog.enabled="$EVENT_LOGGING" \
  15. --conf spark.sql.session.timeZone=UTC \
  16. --conf spark.sql.extensions=com.datastax.spark.connector.CassandraSparkExtensions \
  17. --packages com.datastax.spark:spark-cassandra-connector_2.12:3.2.0 \
  18. ./src/spark/main_partition.py