more merging algorithms, add bench script

config/db/db_read_csv_txs.py

@@ -3,7 +3,7 @@ from cassandra.query import BoundStatement, BatchStatement
 import csv
 
 
-def db_insert_csv_txs(config, tx_file):
+def db_insert_csv_txs(config, tx_file, skip=0, limit=-1):
     print(" == DB TX INSERTION SCRIPT == ")
 
     print(
@@ -13,18 +13,32 @@ def db_insert_csv_txs(config, tx_file):
     session = cluster.connect(config['cassandra_keyspace'])
     print(f"Connection OK")
 
-    with open(tx_file, newline='') as tx_csv:
+    statement = session.prepare(
+            f"INSERT INTO {config['tx_table_name']} (tx_id,address,value,tx_hash,block_id,timestamp) VALUES(?,?,?,?,?,?);")
+    boundStatement = BoundStatement(statement)
+
+    with open(tx_file, newline='') as (tx_csv):
         rowreader = csv.reader(tx_csv, dialect="excel")
         next(rowreader)  # skip header
 
-        statement = session.prepare(
-            f"INSERT INTO {config['tx_table_name']} (tx_id,address,value,tx_hash,block_id,timestamp) VALUES(?,?,?,?,?,?);")
-        boundStatement = BoundStatement(statement)
         batchStatement = BatchStatement()
 
-        for row in rowreader:
+        batch_count = 0
+
+        for i, row in enumerate(rowreader):
+            if i < skip:
+                continue
+            if i == limit:
+                break
+
             batchStatement.add(boundStatement.bind(
                 [int(row[0]), str(row[1]), int(row[2]), str(row[3]), int(row[4]), int(row[5])]))
+            batch_count += 1
+            if batch_count > 256:
+                session.execute(batchStatement)
+                batchStatement = BatchStatement()
+                batch_count = 0
+
         session.execute(batchStatement)
 
     print("Done!")

config/db/tables/transactions/CREATE.sql

@@ -1,9 +1,9 @@
 CREATE TABLE transactions(
-    tx_id INT,
+    tx_id bigint,
     address TEXT,
-    value INT,
+    value bigint,
     tx_hash TEXT,
-    block_id INT,
+    block_id bigint,
     timestamp TIMESTAMP,
     PRIMARY KEY (tx_id, address)
 ) WITH CLUSTERING ORDER BY (address DESC);