progress on mapping data, finding clusters, probably inefficient

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python.analysis.extraPaths": [
+        "./config/db"
+    ]
+}

clean.py

@@ -0,0 +1,9 @@
+import sys
+import json
+
+sys.path.append("config/db")
+
+from db_teardown import db_teardown
+
+config = json.load(open("./settings.json"))
+db_teardown(config)

config/db/db_read_csv_txs.py

@@ -0,0 +1,31 @@
+from cassandra.cluster import Cluster
+from cassandra.query import BoundStatement, BatchStatement
+import csv
+
+
+def db_insert_csv_txs(config, tx_file):
+    print(" == DB TX INSERTION SCRIPT == ")
+
+    print(
+        f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}")
+    cluster = Cluster(config['cassandra_addresses'],
+                      port=config['cassandra_port'])
+    session = cluster.connect(config['cassandra_keyspace'])
+    print(f"Connection OK")
+
+    with open(tx_file, newline='') as tx_csv:
+        rowreader = csv.reader(tx_csv, dialect="excel")
+        next(rowreader)  # skip header
+
+        statement = session.prepare(
+            f"INSERT INTO {config['tx_table_name']} (tx_id,address,value,tx_hash,block_id,timestamp) VALUES(?,?,?,?,?,?);")
+        boundStatement = BoundStatement(statement)
+        batchStatement = BatchStatement()
+
+        for row in rowreader:
+            batchStatement.add(boundStatement.bind(
+                [int(row[0]), str(row[1]), int(row[2]), str(row[3]), int(row[4]), int(row[5])]))
+        session.execute(batchStatement)
+
+    print("Done!")
+    cluster.shutdown()

config/db/db_setup.py

@@ -0,0 +1,25 @@
+from cassandra.cluster import Cluster
+import os
+
+def db_setup(config):
+    print(" == DB SETUP SCRIPT == ")
+
+    print(f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}")
+    cluster  = Cluster(config['cassandra_addresses'], port=config['cassandra_port'])
+    session = cluster.connect()
+    print(f"Connection OK")
+
+    print(f"Creating KEYSPACE '{config['cassandra_keyspace']}'")
+    with open(f"{config['setup_keyspace_dir']}/CREATE.sql") as keyspace_create:
+        session.execute(keyspace_create.read())
+
+    with open(f"{config['setup_keyspace_dir']}/USE.sql") as keyspace_use:
+        session.execute(keyspace_use.read())
+
+    for folder_name in os.listdir(config['setup_tables_dir']):
+        print(f"Creating TABLE '{folder_name}'")
+        with open(f"{config['setup_tables_dir']}/{folder_name}/CREATE.sql") as sql_create:
+            session.execute(sql_create.read())
+
+    print("Done!")
+    cluster.shutdown()

config/db/db_teardown.py

@@ -0,0 +1,22 @@
+from cassandra.cluster import Cluster
+import os
+
+def db_teardown(config):
+    print(" == DB TEARDOWN SCRIPT == ")
+
+    print(f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}")
+    cluster  = Cluster(config['cassandra_addresses'], port=config['cassandra_port'])
+    session = cluster.connect(config['cassandra_keyspace'])
+    print(f"Connection OK")
+
+    for folder_name in os.listdir(config['setup_tables_dir']):
+        print(f"Dropping TABLE '{folder_name}'")
+        with open(f"{config['setup_tables_dir']}/{folder_name}/DROP.sql") as table_drop:
+            session.execute(table_drop.read())
+
+    with open(f"{config['setup_keyspace_dir']}/DROP.sql") as keyspace_drop:
+        print(f"Dropping KEYSPACE '{config['cassandra_keyspace']}'")
+        session.execute(keyspace_drop.read())
+
+    print("Done!")
+    cluster.shutdown()

setup/db/keyspace/CREATE.sql → config/db/keyspace/CREATE.sql

@@ -1 +1 @@
-CREATE KEYSPACE IF NOT EXISTS distributedunionfind WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 3};
+CREATE KEYSPACE IF NOT EXISTS distributedunionfind WITH replication = {'class':'SimpleStrategy', 'replication_factor': 1};

config/db/keyspace/DROP.sql

@@ -0,0 +1 @@
+DROP KEYSPACE IF EXISTS distributedunionfind;

config/db/tables/clusters/CREATE.sql

@@ -0,0 +1,5 @@
+CREATE TABLE clusters(
+    address TEXT,
+    parent TEXT,
+    PRIMARY KEY (address)
+);

config/db/tables/clusters/DROP.sql

@@ -0,0 +1 @@
+DROP TABLE clusters;

config/db/tables/transactions/DROP.sql

@@ -0,0 +1 @@
+DROP TABLE transactions;

scratchpad.py

@@ -0,0 +1,32 @@
+import sys
+import json
+from cassandra.cluster import Cluster
+
+sys.path.append("config/db")
+
+config = json.load(open("./settings.json"))
+
+print(
+    f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}")
+cluster = Cluster(config['cassandra_addresses'],
+                    port=config['cassandra_port'])
+session = cluster.connect(config['cassandra_keyspace'])
+print(f"Connection OK")
+
+result = session.execute("SELECT * FROM clusters")
+print(result.all())
+
+
+"""
+sc = pyspark.SparkContext('spark://osboxes:7077')
+
+data = sc.parallelize(list("aaa bbb cc dd e f"))
+counts = data \
+    .map(lambda x: (x, 1)) \
+    .reduceByKey(add) \
+    .sortBy(lambda x: x[1], ascending=False) \
+    .collect()
+
+for (word, count) in counts:
+    print("{}: {}".format(word, count))
+"""

settings.json

@@ -0,0 +1,13 @@
+{
+    "cassandra_addresses": ["127.0.0.1"],
+    "cassandra_port": 9042,
+    "cassandra_keyspace": "distributedunionfind",
+
+    "setup_db_dir": "config/db",
+    "setup_tables_dir": "config/db/tables",
+    "setup_keyspace_dir": "config/db/keyspace",
+
+    "tx_table_name": "transactions",
+    "clusters_table_name": "clusters"
+
+}

settings.py

@@ -1,6 +0,0 @@
-cassandra_addresses = ['127.0.0.1']
-cassandra_port = 9042
-cassandra_keyspace = 'distributedunionfind'
-
-setup_db_dir = 'setup/db'
-setup_tables_dir = setup_db_dir+"/tables"

setup.py

@@ -1,24 +1,12 @@
-from settings import cassandra_addresses, cassandra_port, setup_db_dir, setup_tables_dir
-from cassandra.cluster import Cluster
-import os
+import sys
+import json
 
-print(" == DB SETUP SCRIPT == ")
+sys.path.append("config/db")
 
-print(f"Attempting Cassandra connection @ {cassandra_addresses}:{cassandra_port}")
-cluster  = Cluster(cassandra_addresses, port=cassandra_port)
-session = cluster.connect()
-print(f"Connection OK")
+from db_setup import db_setup
+from db_read_csv_txs import db_insert_csv_txs
 
-with open(f"{setup_db_dir}/keyspace/CREATE.sql") as keyspace_create:
-    session.execute(keyspace_create.read())
+config = json.load(open("./settings.json"))
 
-with open(f"{setup_db_dir}/keyspace/USE.sql") as keyspace_use:
-    session.execute(keyspace_use.read())
-
-for folder_name in os.listdir(setup_tables_dir):
-    with open(f'{setup_tables_dir}/{folder_name}/CREATE.sql') as sql_create:
-        session.execute(sql_create.read())
-
-session.execute("INSERT INTO transactions (tx_id,address,value,tx_hash,block_id,timestamp) VALUES(1697,'t1KmCvfPMgfQXeNosFqzAmvYdEoYfdnxnVA',15701,'18c23345908f5097456c5f0014411381fd9866790aa65b863aab24ee17453732',818,1477724947)")
-res = session.execute('SELECT * FROM transactions')
-print(res.one())
+db_setup(config)
+db_insert_csv_txs(config, "./small_test_data.csv")

setup/db/keyspace/DROP.sql

@@ -1 +0,0 @@
-DROP KEYSPACE IF EXISTS distributedunionfind

setup/db/tables/transactions/DROP.sql

@@ -1 +0,0 @@
-DROP TABLE transactions

small_test_data.csv

@@ -0,0 +1,181 @@
+tx_id,address,value,tx_hash,block_id,timestamp
+0,18XMcuNcPpUGLEgFogaXqZQWidNCyuWvrJ,4609,2179241ef47c59f043e8e5bfba33d4ccdcbad9bdfc245fab3f1ac0a0cf4bb95d,0,1477724947
+1,12yriKe117c24oqxoapTzWXwb2nPHbHVVi,442,bff57c7602bb55cfdfc07dd2e10274f4b82d6e0fb4eb311b3b06e74c38e61ff2,1,1477725342
+1,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2041,bff57c7602bb55cfdfc07dd2e10274f4b82d6e0fb4eb311b3b06e74c38e61ff2,1,1477725342
+2,1PzCy6TKe4GRn41WoQ9KQ4sfGL8kwSeAvj,3556,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635
+2,1FTkenQqmxtuhNATrV9evtkjfdo54L7hPN,3529,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635
+2,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,788,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635
+2,148Ebq2qHioHGFruAHvSHsut25L5F26TjG,1621,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635
+2,1FLV1Y8jem2G3Cxkh127sQnovaQvQCTtje,1915,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635
+3,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4362,6305c45584d6f8e3daa1f4075eccf24757382641f402e1d8714286d1f49c1ffe,3,1477725636
+4,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,3028,c8f86fe8bdbaf231a03b395e4546e4c4e42ddbab3c2db479945592c0375a87b6,4,1477725729
+5,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2351,544a8690110099a4a1dc6831d6666199019e28cc1c7752ca965201d291c474ad,5,1477726050
+6,16SKLHfQdpF9DUzRFVpZ3RtDi1gViyHiby,1340,0cc5b12d22940ed0a596bf48ab84041c40ec5c35d8df32eea752857db296976f,6,1477726421
+6,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4668,0cc5b12d22940ed0a596bf48ab84041c40ec5c35d8df32eea752857db296976f,6,1477726421
+7,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4141,2b1fb25421601bd23d27e6b6d72331f713f44b6e7f15f2742f73322a31fc4f3a,7,1477726449
+8,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,422,3060c805f22cd47675d0cac8d4bd10839f6e9845cabe1e19c299e6cdee1b50a3,8,1477726589
+9,1K2KwATcG2vQ1L9s1qrLMci1oAMgTa1BYy,503,c0171e55ba1bdd2761e6acd80c7323255ee78d62d89d7dc95ac669b9d084e572,9,1477726862
+9,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,3737,c0171e55ba1bdd2761e6acd80c7323255ee78d62d89d7dc95ac669b9d084e572,9,1477726862
+10,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2981,7a93895f31104c5a3b21f0e32206d98f436fc5f652238e3965790efd80dc9c38,10,1477727261
+11,1Gri49ZrZ87JiD3yBXQZnYUdr8GV9z4MUH,1599,6c6cb74d5f0923bdee2b5e944dde20c8ecd5282e1708bb776a6cc847db2d531d,11,1477727495
+11,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4083,6c6cb74d5f0923bdee2b5e944dde20c8ecd5282e1708bb776a6cc847db2d531d,11,1477727495
+12,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4952,939a8f85fe523bb007898277e5e5e76ee673e252240cac5ad9f42cbefd4a3cf7,12,1477727563
+13,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,1824,a38382874101de402ea61a5a6b1d98ce370089d950df43d951f3c9dd53ed596b,13,1477727805
+14,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,1229,2da65b34c5c809743fe305b6819f19c7cc1f12137beafbc5836e4913a0184554,14,1477728074
+15,1Lzy6HQGL2TEpASEHSgmtvfut9auSkbmJs,3921,a9daae52ab4c501cef3bcf70e372cca192cfe4f0f9277f46f5ff4671e2a10de6,15,1477728463
+15,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2818,a9daae52ab4c501cef3bcf70e372cca192cfe4f0f9277f46f5ff4671e2a10de6,15,1477728463
+16,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2721,ecaea7121fedb14e90595a1bd7b54f8fd677c92266a3ebae361c3f8d8420aa4e,16,1477728645
+17,18hnWNqw8iqnkGg3h5592649XKQux7GUa6,3198,0d13804cd20bf131b7a9e104fc413a7e1b4e48fbfbdb99a017ee1ef6f573e8c7,17,1477728672
+17,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4447,0d13804cd20bf131b7a9e104fc413a7e1b4e48fbfbdb99a017ee1ef6f573e8c7,17,1477728672
+18,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4226,09e2dabcf5a70bb0db27d0d1821c6fa5b1845ca3a02dd45285f7aae2459866c7,18,1477729019
+19,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2944,7bd825c8f456ff67511470d4ae996ae25e7badc7fe0a4b4c3866924062d4a0e0,19,1477729205
+20,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1665,5d70eee42a1e18cc17bc09e8d34754cca35d435b7d775cedffce9d0c1c7dbb0d,20,1477729336
+21,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,2386,ca80a3747917dc81c0f9a90de2185c66d71b704465b4de0aafa721f2763f387c,21,1477729702
+22,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3283,c5059fce2d1d3c2e8d7f31e5e4ce613ecc5db845fd47c8ca46b97dc266aa9dec,22,1477730093
+22,3Dm2pFiNk1PAjxHMuTrPUM8T5a1wxphAkP,1600,c5059fce2d1d3c2e8d7f31e5e4ce613ecc5db845fd47c8ca46b97dc266aa9dec,22,1477730093
+22,3QdE6N6dRdQNUVFy8Vv3twHuQRaa5kvu2U,1577,c5059fce2d1d3c2e8d7f31e5e4ce613ecc5db845fd47c8ca46b97dc266aa9dec,22,1477730093
+23,3QdE6N6dRdQNUVFy8Vv3twHuQRaa5kvu2U,2424,2276f51c7e33c5261fbc53104b743f15bbea2ad46722ac8592f118606ea944f1,23,1477730144
+24,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,368,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327
+24,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,233,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327
+24,3CZQg1dc1HJ41QJsxViuqxpJLXHGvWdy1w,1389,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327
+24,39TwnPh7pEr99ZyYfkUrwLkcTGmrVSBC3R,4526,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327
+24,3PcKS338Y2Je6uEZ77djmQJLChjdaLv75E,2779,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327
+25,3CZQg1dc1HJ41QJsxViuqxpJLXHGvWdy1w,2848,b373d4a299942f628e68f7e7e84fdf4e3b5949f7592299e565a4d0e26bcec64d,25,1477730543
+26,3ESVZJmNGNDziUckU6qUDMmp6ue3v3ErfP,676,2808b24fd80ab192aa7afc55eb0bb31c33bacf1221aef543153ef677799c2ac6,26,1477730761
+26,3CZQg1dc1HJ41QJsxViuqxpJLXHGvWdy1w,2536,2808b24fd80ab192aa7afc55eb0bb31c33bacf1221aef543153ef677799c2ac6,26,1477730761
+27,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,3610,51adf572ffab1de3047f7bc46eff0a6708a62160aaef3cd579e3f8b2e8b9c2cf,27,1477730852
+28,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,688,ac63df67840aaf9a03f12b91a74961cff82609cec8357d8ab223d3153daa478a,28,1477730889
+29,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,3971,acd524b63d166835aeee515126e55670633bffc405ab387789e75c7dfb4fe3eb,29,1477731038
+30,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4928,1f642eff540fe878c3acb1518cd006f55797944b923643790222b85dd4ec3768,30,1477731296
+30,3EXdRFUCu5qBNtLwpWy8NvpBRdNPRsEvXQ,3307,1f642eff540fe878c3acb1518cd006f55797944b923643790222b85dd4ec3768,30,1477731296
+31,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,1653,d3d5711d8b92437837db82331a4132e658922381afef5f60cc6b415ce1999e2d,31,1477731611
+32,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1486,4e690c8576cf921bac42befbfd55f2dcac5f6ea6763ab9b22deca052e15cbbb7,32,1477731703
+33,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1824,cdd555b0512bf1b23828de121ff22b1fc96b81c125c0fd78c51a75b787fcfff2,33,1477731720
+33,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,2108,cdd555b0512bf1b23828de121ff22b1fc96b81c125c0fd78c51a75b787fcfff2,33,1477731720
+34,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1362,e937af274b25f12cca28cb7bd27dab1621a2ca3d9907481cd56b4f942562bf78,34,1477732036
+35,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2341,a65c8de4f5d20706bd4109ad2662a0b36f022c52ede0004bb5fec131fb5cee98,35,1477732212
+35,34oeKULDnRqsiTToVXzYxYLwAuyi1QhTGD,837,a65c8de4f5d20706bd4109ad2662a0b36f022c52ede0004bb5fec131fb5cee98,35,1477732212
+35,32Yh4hKDiKP2h5jRX57HDSNsTJtpryCB8p,3671,a65c8de4f5d20706bd4109ad2662a0b36f022c52ede0004bb5fec131fb5cee98,35,1477732212
+36,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4933,127fa2027adf318c4ed8251b9a8310b20a00cbf83c39dc9ead67fc3561be5b43,36,1477732568
+37,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,1058,f1b9523ddc5574dbe98bfb6f897d3ff030b8873dd877bc230dc491669323618f,37,1477732824
+38,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,679,3514cece33629552536dc2c85fef8f9001ce64cceef3d00d8579458382dbe7e6,38,1477732916
+38,3LyMyw5q3VDmdc5y2uK52SxTFV6YtDRjEU,4161,3514cece33629552536dc2c85fef8f9001ce64cceef3d00d8579458382dbe7e6,38,1477732916
+39,3PCDL1umA9hXMp2YRDurFkQ8tMEjePyQyq,4510,a2e069ab489c5ed32cfbbfbf8c41f4e81d516991666338b1489dec404e1b5219,39,1477733228
+39,338DY888RLprEK6Sj92Ro1X4jspdF9WMTK,2349,a2e069ab489c5ed32cfbbfbf8c41f4e81d516991666338b1489dec404e1b5219,39,1477733228
+39,3FAH22bWvGHbD5xwdAviPD2NkypCcNtZfz,533,a2e069ab489c5ed32cfbbfbf8c41f4e81d516991666338b1489dec404e1b5219,39,1477733228
+39,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1250,a2e069ab489c5ed32cfbbfbf8c41f4e81d516991666338b1489dec404e1b5219,39,1477733228
+40,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,519,55143f25c59c95b9b66e1afb758589415a2e850d9ba3ffc5850ad5ffeeb248f4,40,1477733259
+41,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1022,12df97e178db9f13137eaf879c53f7df7464a67c69e6b418c09c8335af4d8a15,41,1477733269
+41,1NW3ovVf9DfpczHZ7YHajfXgjcN1BBxE8P,223,12df97e178db9f13137eaf879c53f7df7464a67c69e6b418c09c8335af4d8a15,41,1477733269
+42,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,1087,c6b9d1fdb7f8c20f4c578eebb2552b0adbbd1b28a957a8f56eca67de9ebdbb97,42,1477733283
+43,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,440,1b151c9fc241f6296b965c304dc3429786e57b0fb76c5cca351f60d3b531ebf9,43,1477733329
+44,3Gzq61vioahFFfyQR9A9nDt5UM61Jj9hJ9,2318,dee3d416d8beb94e9e170934fb29ad517cfef594fd078704da93ad74dfc9dad0,44,1477733382
+44,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4250,dee3d416d8beb94e9e170934fb29ad517cfef594fd078704da93ad74dfc9dad0,44,1477733382
+44,3GdURTjkyMgFnPjELgWTuVG1aLas3BhShA,3504,dee3d416d8beb94e9e170934fb29ad517cfef594fd078704da93ad74dfc9dad0,44,1477733382
+45,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4543,a4327bf59e02f3cebb63c25d585fca2ef723512a2a891a59890667f178fcc76c,45,1477733666
+46,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2831,6469cc94343c35bd3a9a316990f93c2f5a55806896aaf1be71b6cf3ba4f71295,46,1477733884
+47,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1009,459054b60172dd81dd49930e805827c46b43bd68b8ea12dfc11a307183f58bf3,47,1477734056
+48,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,4911,ded460c087bd6571a1cb0c4376258731a5769aaf56327ab77f418aebd55763f8,48,1477734388
+49,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,224,2d737e75a0cce777212d084d19450ac1396de684c81b48839246841e5cc05c84,49,1477734608
+50,1JNLN6LRiSrx5FWQ9dAtBgdPSo58TZREVc,514,17f64303dfba6f6d1b0c3faeeefaeaddb68c786742d449a1c2ddf8165582988b,50,1477734878
+50,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3338,17f64303dfba6f6d1b0c3faeeefaeaddb68c786742d449a1c2ddf8165582988b,50,1477734878
+50,1AjxAJKs12g16o5gx3YbfTJT197FJkq8Gd,2477,17f64303dfba6f6d1b0c3faeeefaeaddb68c786742d449a1c2ddf8165582988b,50,1477734878
+51,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,3960,99e53ce74a805ff8ce409fcb662b72c6783c2626a7d30f4a89c6675b9bee8281,51,1477735272
+52,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,2305,a05bb346e38e3906dee3387196bb0198319ac38d8a6c24ea9f159641c82ef549,52,1477735521
+53,19C6SDKcCXFZBPtcz5n857EjHKFJ9mSUAe,2904,76fca1eb2b263270dd412c75e7c83b1f3cc023ed9e60b55611ca6e8513f6a70b,53,1477735837
+53,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,4439,76fca1eb2b263270dd412c75e7c83b1f3cc023ed9e60b55611ca6e8513f6a70b,53,1477735837
+53,197uzNAN1bmRrQALTMWrh45HCA91LnAFDL,1650,76fca1eb2b263270dd412c75e7c83b1f3cc023ed9e60b55611ca6e8513f6a70b,53,1477735837
+54,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4407,b2d4e3a43827e1fd9279e2cefd9ce1df75b8da0c80cad131f4105c6f3c2e5e23,54,1477735861
+55,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,566,993544404fb7c7a12937c644c47f778670dd6ef3e57938809667f2794ca0f85d,55,1477735915
+56,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2603,ba75c0a7c4a5d1ef9b705f8309f415bc085354c9d8e2811d27919a31f33f7952,56,1477736274
+57,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2021,40680e7d122a5c24d0b14a5a6b3ad32322dd5aa904fccf6fc501fdaa0ae92c55,57,1477736499
+58,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2564,cf2fc800424cad5f6e6ca9e6fe867610769e6380f05cbf9d25c5af957a6565af,58,1477736684
+59,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3689,b87f98cd451fac55a912c27f02ef74401e8acd93419ca1d67d2b1a27eff7b42c,59,1477736774
+60,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,998,46591437f61e98a5c50146ba2166b41bebc2d6b73b12f2743ec1e3054fe79023,60,1477736878
+61,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4415,766970297f7cf1359ce38a146add3b8c4dc34ab9d9ee1913feb262cb2cfb41d1,61,1477736989
+62,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,991,4053ec2da3e43b45abf6f4001c6b88753e11818c172bf74d895679ebdce107bf,62,1477737035
+63,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3660,a39ef4617c802782d6e96fbb66d38f637e9937df810ea178262933e41cdbc5b8,63,1477737405
+64,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,815,e15a1524bfb8e476db962bb69870411cfad1154a071f1c06aaca0798da54dda3,64,1477737724
+65,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,817,36fc7f8881f09f7fa968bbde9d74cc0ceb9ea3d404ebc111694876bd2c863bb4,65,1477737939
+65,1JMgMQMK9JC9CBodvjxvLtoE42fgreGbHP,3649,36fc7f8881f09f7fa968bbde9d74cc0ceb9ea3d404ebc111694876bd2c863bb4,65,1477737939
+65,1QEGqswF6jcNmxG5Sejhf5puNpaQ27HBDX,4661,36fc7f8881f09f7fa968bbde9d74cc0ceb9ea3d404ebc111694876bd2c863bb4,65,1477737939
+66,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,4886,c4b4b659cb94522fb2143c04d4a6fa4414b2e5aac6594f3e5251585e50332afa,66,1477738165
+67,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3780,763f8c9b311c0f2ba49cc4f20ee70704445165de6efaa371aeccc0a756cb6a90,67,1477738196
+67,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,4369,763f8c9b311c0f2ba49cc4f20ee70704445165de6efaa371aeccc0a756cb6a90,67,1477738196
+68,1PmCGRS6Crrfz2QcmT3XUGcXtvLj2cxUJR,4157,6989ef40065760ff1318c49c1447d1b8f1da9c2c729f895b91215c2851e31b2c,68,1477738542
+68,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,3383,6989ef40065760ff1318c49c1447d1b8f1da9c2c729f895b91215c2851e31b2c,68,1477738542
+69,13S8ipxJvzz22xKCrHfe3MWC6HTDR8gYhS,1739,58b1c24524e5fd629d29389aff989bcc90c2533c7e1401c065faed12ff3a5233,69,1477738636
+69,1NEkBXK77FNciK2BwqA55diDUhgsvjyh5V,768,58b1c24524e5fd629d29389aff989bcc90c2533c7e1401c065faed12ff3a5233,69,1477738636
+69,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,2987,58b1c24524e5fd629d29389aff989bcc90c2533c7e1401c065faed12ff3a5233,69,1477738636
+70,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2281,52b2efe12c32076f18830b70d228bfa5e56bfd7119546c94779ec9a16f32bf68,70,1477739005
+71,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2551,8fe146a5e1a141e9b2bb0dbf2aa0a25a2fb00974a47b9c5ed8a78f671c64c9e4,71,1477739028
+71,1EksR7DeEodWTBwoPmzbmfFYt1fAMzD2RU,2613,8fe146a5e1a141e9b2bb0dbf2aa0a25a2fb00974a47b9c5ed8a78f671c64c9e4,71,1477739028
+71,17pL4RA7GwV87mjLQRuy4URUpBRhBpAUfx,1461,8fe146a5e1a141e9b2bb0dbf2aa0a25a2fb00974a47b9c5ed8a78f671c64c9e4,71,1477739028
+71,15YK8ZZyL7YezuwpraQodjF56pws7TRjd2,70,8fe146a5e1a141e9b2bb0dbf2aa0a25a2fb00974a47b9c5ed8a78f671c64c9e4,71,1477739028
+72,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3595,e2364f281a207b154678561f5c29f2bea911940445b4446f3bd3e98be2b80719,72,1477739078
+73,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3634,a646b1b87b24916cdf6ce439d590cf4786c33a877ef7442aa379b05fe938edf0,73,1477739116
+74,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,604,8bd3f57789e48effe34db65627ae5759c7be8b3290ab16e89e0711d3591c8be9,74,1477739120
+75,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4347,d48bb09aa5b8b85df707be1d8de322d3a282353a694335745befc76c6bcab09e,75,1477739348
+76,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2597,80927bf38ad2a17c63f8bcad914696adc1146d822a4fd9abaec4812f0102de41,76,1477739562
+77,1Gtcs8i8SkQNWAwxi1WzLFPWmt7JT72n7n,352,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797
+77,1BBeBwTNJmWRFcmfpT36Xzff19mc65Ncnu,3039,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797
+77,1AAWJ3CKn2eaM6PHffukNunPmARB7kDsqU,3605,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797
+77,16fdPvFnSFYpQEWqPXG3U5rmm78YZbiuXp,2792,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797
+77,17aoiN8dc8sk55EKFQtTQ31XhJHAbHWrG,4116,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797
+78,1JheBVxaY5ZyWMgDBe4KdG2DPQrS1sVvKv,3862,58f4dfdea4d9fb9204965469a7d20a1feba6dcf104200bb24bd4f50d9baab5df,78,1477739918
+78,1Fkt2RPovzEUMf2118RJUTRgJYKeEgFwFA,460,58f4dfdea4d9fb9204965469a7d20a1feba6dcf104200bb24bd4f50d9baab5df,78,1477739918
+78,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,1223,58f4dfdea4d9fb9204965469a7d20a1feba6dcf104200bb24bd4f50d9baab5df,78,1477739918
+79,1255XDfmdQTfURAiKGz83bdstfFNyt6vUN,4825,46057443f11993dd2a979548b9fb82329b25fc4cb73500b479e06c8f18712e1e,79,1477740069
+80,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,3755,6fbff7e292b2040cdc7d7dbffa56054f58e7c1bbb64d226c85b2316a50835b90,80,1477740305
+81,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,2656,18cd57e874fc6e8b13a68d6f18c761c0906ca0bf26cdf70b9e88fee405d6afdc,81,1477740420
+82,1Fkt2RPovzEUMf2118RJUTRgJYKeEgFwFA,577,7980fabd136a2f7edd31c7be3e374761efc79e130a656d018706af02af76b04d,82,1477740456
+83,1NoXX6QyKRw3cPnbMC4LP9qiGZdXYcsWR6,3531,60b605e46b3415484377a2e6c748436b15b34385ae4cbc562e38046c9eda296f,83,1477740489
+83,1255XDfmdQTfURAiKGz83bdstfFNyt6vUN,138,60b605e46b3415484377a2e6c748436b15b34385ae4cbc562e38046c9eda296f,83,1477740489
+84,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,2486,323b60dae16e1639d3d97eec2eaeb743d552b7ece6c336999dd3f1b8bbb7a1f6,84,1477740669
+85,1AAWJ3CKn2eaM6PHffukNunPmARB7kDsqU,2287,5285bf90f158d58f4d09a7f42030c7fd7aae557a8463d9dc41d39eda783500f8,85,1477740882
+86,1NoXX6QyKRw3cPnbMC4LP9qiGZdXYcsWR6,2064,64e641282b4cbf0155740c20e36242aec85fef27f6407220a58d2496e4556df4,86,1477741104
+87,1341WZDtruCsQDthA6scJLSkRtojwBAuJu,2743,58ef204b34791846327fe625c8c18a35cd433de40614e494a066b1c02918f1cb,87,1477741388
+88,1JDjYwc8EJViauMFoSt8KZLaWscCW2huKu,2590,fb17f512882b9ca4f78901139947a4586ab45efff93f3c0e62e30275eb0a7046,88,1477741413
+88,1Gtcs8i8SkQNWAwxi1WzLFPWmt7JT72n7n,1209,fb17f512882b9ca4f78901139947a4586ab45efff93f3c0e62e30275eb0a7046,88,1477741413
+88,1BBeBwTNJmWRFcmfpT36Xzff19mc65Ncnu,3320,fb17f512882b9ca4f78901139947a4586ab45efff93f3c0e62e30275eb0a7046,88,1477741413
+88,1EPKRbWGii8rKfBM3weGXx86mSeZBSdc9,1669,fb17f512882b9ca4f78901139947a4586ab45efff93f3c0e62e30275eb0a7046,88,1477741413
+89,1MyujkWgHoqMRxiG7aCrLUTdKgZ9eR1pj4,2914,ff9413176e9a9baa6fb1b6e4548e8799c190b2296ca883b3a4ab98e0aeeb59cf,89,1477741755
+90,1DaVpKn8WyNFspk9eYn7THcJhKwSnAkY2P,3888,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1341WZDtruCsQDthA6scJLSkRtojwBAuJu,2687,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1Gtcs8i8SkQNWAwxi1WzLFPWmt7JT72n7n,51,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1Ai6zMeNeRvKYgkzuMDcYVhn9ounzzvrJM,87,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,18o3Zxu3wZ2udocLDprnDYbjfCj2vfAPVb,256,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1FKkzWTUToYWSrBntn2bNJCWCySVFmmW47,3816,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1973XU2xwYg8mwH6q111zdn2pEEzBSS66F,4081,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1ANYUUEX8HNPEN59RyhfYG3wLbr5zTLET3,2198,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1DUpoFzu85KRWZAVg4YHkeHLPsp5eEWo5E,824,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1MgU7ZSZEacoBsk9FaDnNrr8iLHDCdW3zy,181,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1NoXX6QyKRw3cPnbMC4LP9qiGZdXYcsWR6,3255,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1MyujkWgHoqMRxiG7aCrLUTdKgZ9eR1pj4,2482,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,13LsCQgxkwV1TBagv2Dhd1P93KFgTJjCtL,3502,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1Nk2nNSxY654swupCfWF8XfoJq9pktbChB,725,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+90,1Fv9wFQ7zHPhDykeo3KoLRxWaxwm6r4PZ1,2658,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009
+91,1ANYUUEX8HNPEN59RyhfYG3wLbr5zTLET3,2722,bef4df40757a1bfb0a6ae91a2b7b15eb30aecda80d557722064518b914a1578a,91,1477742144
+92,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,4214,63c8994839b5bef056b655baebb7141d6b569de622d29dcb33d6e855a9cd1a15,92,1477742239
+92,1Fv9wFQ7zHPhDykeo3KoLRxWaxwm6r4PZ1,384,63c8994839b5bef056b655baebb7141d6b569de622d29dcb33d6e855a9cd1a15,92,1477742239
+93,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,2808,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278
+93,1UGpcdT7JwCYVfKcHyvdNEKjSkbdZs3MK,3406,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278
+93,1EzWmRUxdJ2KRddYiMCqCYXQa79V4SVYtK,1584,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278
+93,16KfsnZJqRhUBM4zJigYzDMTrpBCJnFFZc,89,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278
+93,17D77wQF9gQuoX7muR9Mk1xjLBEEmoNrhK,3317,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278
+94,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,2659,30f07a8db3da1f64b676dd82dfded1e2f8416815ca2d98a2a0bdd7da23af8c80,94,1477742635
+95,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,1698,7dfdb42e4f84e5bb2a497fd28020479b1dabd28a985344cc5f577bd9f7c678d4,95,1477743011
+96,1hJz3Dc2f7cPvBspdiY5p1UMsKMA2gogT,3935,d15196c305dd77f2d3e3259223cff1eaf490bffbca9d62ce03abbb08bcf6fd33,96,1477743384
+96,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,4052,d15196c305dd77f2d3e3259223cff1eaf490bffbca9d62ce03abbb08bcf6fd33,96,1477743384
+96,19wz95smVq3AWCcpst8b8CaPT5m2a2Sf2U,3200,d15196c305dd77f2d3e3259223cff1eaf490bffbca9d62ce03abbb08bcf6fd33,96,1477743384
+97,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,4341,63271b55cd4bc253433b66bb559cd55258d44e0353a70c2fe5e13c81f907eff0,97,1477743664
+97,1DS7SAKyEynj8oH5s9PgxE2BUfbNckd355,4525,63271b55cd4bc253433b66bb559cd55258d44e0353a70c2fe5e13c81f907eff0,97,1477743664
+98,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,4786,f82f5d6cc0b8770b8ece9b7fad661412994999921f17a1d0446504ddca87851a,98,1477743799
+99,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,874,870382a095c8364d1da768c6787a44ec3cc15b6fadd5085d85ed8c813cb5da5a,99,1477743873
+99,1BwcTvisQ8TFnVMtva78zWMyvqqr7k4dyV,152,870382a095c8364d1da768c6787a44ec3cc15b6fadd5085d85ed8c813cb5da5a,99,1477743873
+99,1H99UihszQWoppVqWvp5LDjR8PUTFQ58EU,576,870382a095c8364d1da768c6787a44ec3cc15b6fadd5085d85ed8c813cb5da5a,99,1477743873
+100,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,670,d08a6ec49f285c361a63d1c9528b47ef4d2efb3a41a9f6fbf17078dd44c3483f,100,1477743908
+101,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,469,3665474c37ebb5524a3bcceb8249b6c84e1b7fcefe70f1c9db898673a6c84c2e,101,1477744047
+102,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,3200,3134370264311c339b8fb8e1676bf053cec6f18100f5941c2ae4b7ec9cbd3ce8,102,1477744286
+103,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,3538,e80d703ccb172d18fb0a8f09112e540d1b44584410174bcad7a5c1e1658feddb,103,1477744536
+104,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,4073,ef3e9380f2b2eb6693fdda09a5ec9f4329d7b48e91e5f9585b2eee09227fff58,104,1477744928

src/spark/main.py

@@ -0,0 +1,76 @@
+from gc import collect
+from sqlite3 import Row
+from typing import Iterable
+from operator import add
+
+from pyspark.sql import SparkSession
+from pyspark.sql import functions as F
+
+
+
+spark = SparkSession.builder \
+    .appName('SparkCassandraApp') \
+    .config('spark.cassandra.connection.host', 'localhost') \
+    .config('spark.cassandra.connection.port', '9042') \
+    .config('spark.cassandra.output.consistency.level', 'ONE') \
+    .config("spark.sql.extensions",  "com.datastax.spark.connector.CassandraSparkExtensions") \
+    .config('directJoinSetting', 'on') \
+    .master('spark://osboxes:7077') \
+    .getOrCreate()
+
+spark.conf.set("spark.sql.catalog.myCatalog",
+               "com.datastax.spark.connector.datasource.CassandraCatalog")
+
+
+tx_addr_groups = spark.read.table("myCatalog.distributedunionfind.transactions") \
+    .groupBy("tx_id") \
+    .agg(F.collect_set('address').alias('addresses')) \
+    .toLocalIterator()
+
+def insertCluster (row):
+    addrs: Iterable[str] = row['addresses']
+    df = spark.createDataFrame(map(lambda addr: (addr, addrs[0]), addrs), schema=['address', 'parent'])
+
+    df.writeTo("myCatalog.distributedunionfind.clusters").overwrite()
+
+"""
+    tuple structure:
+        Row => Row(parent=addr, addresses=list[addr]
+        Iterable[str] => list[addr]
+"""
+def find(data: tuple[Row, Iterable[str]]):
+    cluster = data[0]
+    tx = data[1]
+
+    clusteraddresses = cluster['addresses'] + [cluster['parent']]
+
+    if any(x in tx for x in clusteraddresses):
+        return cluster['parent']
+    else:
+        return None
+
+for addr_group in tx_addr_groups:
+    clusters_df = spark.read.table("myCatalog.distributedunionfind.clusters")
+
+    clusters = clusters_df \
+        .groupBy("parent") \
+        .agg(F.collect_set('address').alias('addresses'))
+
+    if (clusters.count() == 0):
+        insertCluster(addr_group)
+        continue
+
+    df = clusters.rdd \
+        .map(lambda cluster: (cluster, addr_group['addresses'])) \
+        .map(find) \
+        .filter(lambda x: x != None) \
+        .collect()
+
+    if(len(df) == 0):
+        insertCluster(addr_group)
+        continue
+
+    print(addr_group)
+    print(df)
+
+    break

start_services.sh

@@ -0,0 +1,6 @@
+SPARK_HOME="/home/osboxes/Downloads/spark-3.2.2-bin-hadoop3.2"
+SPARK_MASTER="spark://osboxes:7077"
+
+"$SPARK_HOME"/sbin/start-master.sh
+SPARK_WORKER_INSTANCES=5 "$SPARK_HOME"/sbin/start-worker.sh "$SPARK_MASTER"
+docker run -d -p 9042:9042 cassandra

submit.sh

@@ -0,0 +1,14 @@
+SPARK_HOME="/home/osboxes/Downloads/spark-3.2.2-bin-hadoop3.2"
+MEMORY="4g"
+SPARK_MASTER="spark://osboxes:7077"
+CASSANDRA_HOST="localhost"
+
+
+"$SPARK_HOME"/bin/spark-submit \
+--master "$SPARK_MASTER" \
+--conf spark.executor.memory="$MEMORY" \
+--conf spark.cassandra.connection.host="$CASSANDRA_HOST" \
+--conf spark.sql.session.timeZone=UTC \
+--conf spark.sql.extensions=com.datastax.spark.connector.CassandraSparkExtensions \
+--packages com.datastax.spark:spark-cassandra-connector_2.12:3.2.0 \
+./src/spark/main.py

teardown.py

@@ -1,21 +0,0 @@
-from settings import cassandra_addresses, cassandra_port, cassandra_keyspace, setup_tables_dir, setup_db_dir
-from cassandra.cluster import Cluster
-import os
-
-print(" == DB TEARDOWN SCRIPT == ")
-
-print(f"Attempting Cassandra connection @ {cassandra_addresses}:{cassandra_port}")
-cluster  = Cluster(cassandra_addresses, port=cassandra_port)
-session = cluster.connect(cassandra_keyspace)
-print(f"Connection OK")
-
-for folder_name in os.listdir(setup_tables_dir):
-    print(f"Dropping table {folder_name}")
-    with open(f'{setup_tables_dir}/{folder_name}/DROP.sql') as sql_drop:
-        session.execute(sql_drop.read())
-
-with open(f"{setup_db_dir}/keyspace/DROP.sql") as keyspace_create:
-    print(f"Dropping keyspace {cassandra_keyspace}")
-    session.execute(keyspace_create.read())
-
-print("Done!")