progress on mapping data, finding clusters, probably inefficient

config/db/db_read_csv_txs.py

@@ -0,0 +1,31 @@
+from cassandra.cluster import Cluster
+from cassandra.query import BoundStatement, BatchStatement
+import csv
+
+
+def db_insert_csv_txs(config, tx_file):
+    print(" == DB TX INSERTION SCRIPT == ")
+
+    print(
+        f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}")
+    cluster = Cluster(config['cassandra_addresses'],
+                      port=config['cassandra_port'])
+    session = cluster.connect(config['cassandra_keyspace'])
+    print(f"Connection OK")
+
+    with open(tx_file, newline='') as tx_csv:
+        rowreader = csv.reader(tx_csv, dialect="excel")
+        next(rowreader)  # skip header
+
+        statement = session.prepare(
+            f"INSERT INTO {config['tx_table_name']} (tx_id,address,value,tx_hash,block_id,timestamp) VALUES(?,?,?,?,?,?);")
+        boundStatement = BoundStatement(statement)
+        batchStatement = BatchStatement()
+
+        for row in rowreader:
+            batchStatement.add(boundStatement.bind(
+                [int(row[0]), str(row[1]), int(row[2]), str(row[3]), int(row[4]), int(row[5])]))
+        session.execute(batchStatement)
+
+    print("Done!")
+    cluster.shutdown()

config/db/db_setup.py

@@ -0,0 +1,25 @@
+from cassandra.cluster import Cluster
+import os
+
+def db_setup(config):
+    print(" == DB SETUP SCRIPT == ")
+
+    print(f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}")
+    cluster  = Cluster(config['cassandra_addresses'], port=config['cassandra_port'])
+    session = cluster.connect()
+    print(f"Connection OK")
+
+    print(f"Creating KEYSPACE '{config['cassandra_keyspace']}'")
+    with open(f"{config['setup_keyspace_dir']}/CREATE.sql") as keyspace_create:
+        session.execute(keyspace_create.read())
+
+    with open(f"{config['setup_keyspace_dir']}/USE.sql") as keyspace_use:
+        session.execute(keyspace_use.read())
+
+    for folder_name in os.listdir(config['setup_tables_dir']):
+        print(f"Creating TABLE '{folder_name}'")
+        with open(f"{config['setup_tables_dir']}/{folder_name}/CREATE.sql") as sql_create:
+            session.execute(sql_create.read())
+
+    print("Done!")
+    cluster.shutdown()

config/db/db_teardown.py

@@ -0,0 +1,22 @@
+from cassandra.cluster import Cluster
+import os
+
+def db_teardown(config):
+    print(" == DB TEARDOWN SCRIPT == ")
+
+    print(f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}")
+    cluster  = Cluster(config['cassandra_addresses'], port=config['cassandra_port'])
+    session = cluster.connect(config['cassandra_keyspace'])
+    print(f"Connection OK")
+
+    for folder_name in os.listdir(config['setup_tables_dir']):
+        print(f"Dropping TABLE '{folder_name}'")
+        with open(f"{config['setup_tables_dir']}/{folder_name}/DROP.sql") as table_drop:
+            session.execute(table_drop.read())
+
+    with open(f"{config['setup_keyspace_dir']}/DROP.sql") as keyspace_drop:
+        print(f"Dropping KEYSPACE '{config['cassandra_keyspace']}'")
+        session.execute(keyspace_drop.read())
+
+    print("Done!")
+    cluster.shutdown()

config/db/keyspace/CREATE.sql

@@ -0,0 +1 @@
+CREATE KEYSPACE IF NOT EXISTS distributedunionfind WITH replication = {'class':'SimpleStrategy', 'replication_factor': 1};

config/db/keyspace/DROP.sql

@@ -0,0 +1 @@
+DROP KEYSPACE IF EXISTS distributedunionfind;

config/db/keyspace/USE.sql

@@ -0,0 +1 @@
+USE distributedunionfind;

config/db/tables/clusters/CREATE.sql

@@ -0,0 +1,5 @@
+CREATE TABLE clusters(
+    address TEXT,
+    parent TEXT,
+    PRIMARY KEY (address)
+);

config/db/tables/clusters/DROP.sql

@@ -0,0 +1 @@
+DROP TABLE clusters;

config/db/tables/transactions/CREATE.sql

@@ -0,0 +1,9 @@
+CREATE TABLE transactions(
+    tx_id INT,
+    address TEXT,
+    value INT,
+    tx_hash TEXT,
+    block_id INT,
+    timestamp TIMESTAMP,
+    PRIMARY KEY (tx_id, address)
+) WITH CLUSTERING ORDER BY (address DESC);

config/db/tables/transactions/DROP.sql

@@ -0,0 +1 @@
+DROP TABLE transactions;