From 5b9ec0da6ae35c4fe83426b0e4fcf590fb9c4525 Mon Sep 17 00:00:00 2001 From: nitowa Date: Tue, 23 Aug 2022 15:12:52 -0400 Subject: [PATCH] progress on mapping data, finding clusters, probably inefficient --- .vscode/settings.json | 5 + __pycache__/settings.cpython-310.pyc | Bin 318 -> 451 bytes clean.py | 9 + .../db_read_csv_txs.cpython-310.pyc | Bin 0 -> 1274 bytes .../db/__pycache__/db_setup.cpython-310.pyc | Bin 0 -> 1096 bytes .../__pycache__/db_teardown.cpython-310.pyc | Bin 0 -> 1013 bytes config/db/db_read_csv_txs.py | 31 +++ config/db/db_setup.py | 25 +++ config/db/db_teardown.py | 22 +++ {setup => config}/db/keyspace/CREATE.sql | 2 +- config/db/keyspace/DROP.sql | 1 + {setup => config}/db/keyspace/USE.sql | 0 config/db/tables/clusters/CREATE.sql | 5 + config/db/tables/clusters/DROP.sql | 1 + .../db/tables/transactions/CREATE.sql | 0 config/db/tables/transactions/DROP.sql | 1 + scratchpad.py | 32 ++++ settings.json | 13 ++ settings.py | 6 - setup.py | 28 +-- setup/db/keyspace/DROP.sql | 1 - setup/db/tables/transactions/DROP.sql | 1 - small_test_data.csv | 181 ++++++++++++++++++ src/spark/main.py | 76 ++++++++ start_services.sh | 6 + submit.sh | 14 ++ teardown.py | 21 -- 27 files changed, 431 insertions(+), 50 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 clean.py create mode 100644 config/db/__pycache__/db_read_csv_txs.cpython-310.pyc create mode 100644 config/db/__pycache__/db_setup.cpython-310.pyc create mode 100644 config/db/__pycache__/db_teardown.cpython-310.pyc create mode 100644 config/db/db_read_csv_txs.py create mode 100644 config/db/db_setup.py create mode 100644 config/db/db_teardown.py rename {setup => config}/db/keyspace/CREATE.sql (52%) create mode 100644 config/db/keyspace/DROP.sql rename {setup => config}/db/keyspace/USE.sql (100%) create mode 100644 config/db/tables/clusters/CREATE.sql create mode 100644 config/db/tables/clusters/DROP.sql rename {setup => config}/db/tables/transactions/CREATE.sql (100%) create mode 100644 config/db/tables/transactions/DROP.sql create mode 100644 scratchpad.py create mode 100644 settings.json delete mode 100644 settings.py delete mode 100644 setup/db/keyspace/DROP.sql delete mode 100644 setup/db/tables/transactions/DROP.sql create mode 100644 small_test_data.csv create mode 100644 src/spark/main.py create mode 100755 start_services.sh create mode 100755 submit.sh delete mode 100644 teardown.py diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..431db64 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python.analysis.extraPaths": [ + "./config/db" + ] +} \ No newline at end of file diff --git a/__pycache__/settings.cpython-310.pyc b/__pycache__/settings.cpython-310.pyc index f8260baabef8699192ef6719b52d2484862d6b6c..f140811c411aee6c6555108091847d723e7c8d61 100644 GIT binary patch delta 279 zcmdnTbeNehpO=@50SL0@{!21ooX96*patZmFr+g^F{Cg?F{Us@F{Lm^F{iLZv81x3 zu!=K8u>xr}AkCJ-9?YQ0G0{&w%%oa9IVZ8WSlvoJI5W2(CpEaFD6u3py;5CALA@xo zASW|9u_QA;FFq|Xxg@_xUBODhxVDO;IJKm-KtCm^ie0}XF)1gtxQbIhJGHX7ATc@B zPm_IOmsN~hGSJw>yp*EEcw}>mq2@>NfX#?cNs3R&EV?BC;Xw=oiwHqPV5Wk^Rx%W^ a0sUCSHt~&%5Fe1m#=yb|hD;ocJd6O?Z&V=w delta 120 zcmX@iypM@5pO=@50SGE={w1wqn8+uiA_wH9Fr+g^F{Cg?F{Us@F{Lm^F{d)8u!u86 zv81pDGib6+j8YfnC{8UYEznO%s$$nKNleN~E%wu7op{}9vLT~9=SqemR-idWtdnyY RUHCvYuraVO0wEI*BLH2886f}w diff --git a/clean.py b/clean.py new file mode 100644 index 0000000..0b177db --- /dev/null +++ b/clean.py @@ -0,0 +1,9 @@ +import sys +import json + +sys.path.append("config/db") + +from db_teardown import db_teardown + +config = json.load(open("./settings.json")) +db_teardown(config) \ No newline at end of file diff --git a/config/db/__pycache__/db_read_csv_txs.cpython-310.pyc b/config/db/__pycache__/db_read_csv_txs.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a32599325d715c1c5dcc0769bb1ead7d662ef19 GIT binary patch literal 1274 zcmY*YOK;mo5Z>imjzrr@n!-R)urI&_l&I~oZPGY$0vK)WBC^|rKp{}vjm3ssGP|o- z7O0cjqtPEw1dxyYOMC6f_nvZbXDM4scDciuZ+2$BeHE?N8iK}3BRR7W`d1t8HUo@@ zFvOQ22qHK`IgT*ax{(>V8JW2iS+#9uc4QNa*ngnNA!ZMS&ILrF5G%KvjfA8;G?f3S z7%|e5v7|X=66{ze!==5S=#tk>lk;J8VTf&vkOT@dC=Y+5Ld4uceu9NfPj>^=$(p9OpSy~j@v_YU@hUiazV&xhJG`E*CZ8I&?*$3Zt1B4&igK~gYA z6PXq)co?W#$+e>&6T+zwRJ_ICD1U{Q1($Hdmj0`oEBqxr6=j^zNvpd`dGLd(?*pSJ zS;}Z+({Vzx5JxUa;|%IfnhUN#`S2i!T5{Z%@gSpp7Uy*GB-jQZCG9IR?bmTOqHVy2 zu^6@oS&_WdZkgs($T%;<;Fp~rj~@57zi%&l_$`N6j)XNtGvtBT0%A7Op7=(!WTLBNx{4BF!k&TRa)%!j=m-j>rKQRE-+EIJqj^ zfJH9eQRENadskq;_pX!)QU4(R8PIyW(-{^y?G$2AjH&4Cra&tl03~wwh(W2lPk?b} z!Kgz9@bz^;UA-^I;$C^GHUUeSpj=*~4rCTzgYd9n+`-nRXE@lo^bHUDhL1gLfn1;- zU=Of&*)%@-&-0s@e*s~kt2URn=ib%o7kj_H!e5OjKV9uGzYURfc)hZB#OL5p-gkLH TMj3se&sumOnrH*4G|m43Js(}$ literal 0 HcmV?d00001 diff --git a/config/db/__pycache__/db_setup.cpython-310.pyc b/config/db/__pycache__/db_setup.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9600bdae31877f20af590c9a8259b53751b15f1d GIT binary patch literal 1096 zcmZ`&Pfy!06t|tU3D83MBTY;kvR$?wAlem;iIR2_Vq%CEO<0<$G&T?|O-h^!s9I?f zyH5K6s8V~}H`!;%b*FuWUbbhqwB3O?ezt$_{rr2+iKBde5yA7|6YrOX&~Leze+Vr0 zp$iKPLktI~i7Cc%P69#+)0qAZQH_x*DrWvb6e{A>aDui-czo<(iA){?#s%mObV0$B zz=rp*8H}Whn6`-?5Yu6nk+Z(8e0b~`qb&QK^l%ps$Po1i@QexH1pk8KFZMNsX@D## zB-bbR_{U05>ym-q)dra%8iML4C_;yJuVECLZ;&{=0aUhg=*f(qpYV0X*PrD3|M9Z} z12S0N1g$dqyxq${wkzEXR5+g@_yH*>JSQuLJhMtC`=m~PUgx?E`>x80;Z)2aWX=$L zinZg;Giz_pDj!(YO6}~#sye5KCp8ImUY3%CH(QAxURsVTgd4KhwY(?{xtI7+Xzg2c z-FtM@UB+TA1Q#Q`OP6POD~gjMrn($Ed1prdc+^>UV(uz|qsn>pq~uhrS9ER0{lf1= z%k}sO11xybZq*;5I`iYslI@&UO0~+4xDGm|eO6U7S{R2WZX@7A1s0}4wbH={P#Iyn zlPO0be^FdWGp*PUlUV-6)R+{Q>QRe_)Zn+=YbQL_Aybx;NTf#Kiv-lEDX!WHi*CZA zk!FE*;a>uM{7X{?ewIkk4q> zzKWXMjzlB6<-#t5cI-DGBes1O!v5Ru{gByXb(=Ndt*h#HT6d}WxX|TQdJV?JTVw;9 z&^BN!;H|$KnzUoO%u`?-5ol-3Cr;1TY&Y3(oQuA#yV(oyx Olh;~g!Q?gOG~zmqiPAD@Vgdwgh)PqH#s;FLaj;zhRV!^` z*J*#igjA0Emwer6{{n|?&neJe=ET3g_rCAF=QzAroI%jeLK4IlLch#rx)IoHLYFQK z6j2{4MJ3+A~?|b-Ttz>vEeqi+@tOK zL38hT*F9_;yxl)CXn(b?6ze7`jL+PLFQp$-;kyBkV-~28$L^-97lTJb+ox18DVe;- zFLh~xCtN7_XttU4!UX@su4Uo}tY2(QB=>f7Su)j2+RE=lOtxt#`sJqJNn+ACs&Bp9 zc3<3Mc)fn!YaZW^a=uviAS ss;%drzDez&kn~hU3CyWan&i|_0fTm4bvf-t?6vs^rTO^^5VJu30k$a&_y7O^ literal 0 HcmV?d00001 diff --git a/config/db/db_read_csv_txs.py b/config/db/db_read_csv_txs.py new file mode 100644 index 0000000..2a714f7 --- /dev/null +++ b/config/db/db_read_csv_txs.py @@ -0,0 +1,31 @@ +from cassandra.cluster import Cluster +from cassandra.query import BoundStatement, BatchStatement +import csv + + +def db_insert_csv_txs(config, tx_file): + print(" == DB TX INSERTION SCRIPT == ") + + print( + f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}") + cluster = Cluster(config['cassandra_addresses'], + port=config['cassandra_port']) + session = cluster.connect(config['cassandra_keyspace']) + print(f"Connection OK") + + with open(tx_file, newline='') as tx_csv: + rowreader = csv.reader(tx_csv, dialect="excel") + next(rowreader) # skip header + + statement = session.prepare( + f"INSERT INTO {config['tx_table_name']} (tx_id,address,value,tx_hash,block_id,timestamp) VALUES(?,?,?,?,?,?);") + boundStatement = BoundStatement(statement) + batchStatement = BatchStatement() + + for row in rowreader: + batchStatement.add(boundStatement.bind( + [int(row[0]), str(row[1]), int(row[2]), str(row[3]), int(row[4]), int(row[5])])) + session.execute(batchStatement) + + print("Done!") + cluster.shutdown() diff --git a/config/db/db_setup.py b/config/db/db_setup.py new file mode 100644 index 0000000..a9b4892 --- /dev/null +++ b/config/db/db_setup.py @@ -0,0 +1,25 @@ +from cassandra.cluster import Cluster +import os + +def db_setup(config): + print(" == DB SETUP SCRIPT == ") + + print(f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}") + cluster = Cluster(config['cassandra_addresses'], port=config['cassandra_port']) + session = cluster.connect() + print(f"Connection OK") + + print(f"Creating KEYSPACE '{config['cassandra_keyspace']}'") + with open(f"{config['setup_keyspace_dir']}/CREATE.sql") as keyspace_create: + session.execute(keyspace_create.read()) + + with open(f"{config['setup_keyspace_dir']}/USE.sql") as keyspace_use: + session.execute(keyspace_use.read()) + + for folder_name in os.listdir(config['setup_tables_dir']): + print(f"Creating TABLE '{folder_name}'") + with open(f"{config['setup_tables_dir']}/{folder_name}/CREATE.sql") as sql_create: + session.execute(sql_create.read()) + + print("Done!") + cluster.shutdown() \ No newline at end of file diff --git a/config/db/db_teardown.py b/config/db/db_teardown.py new file mode 100644 index 0000000..faf865f --- /dev/null +++ b/config/db/db_teardown.py @@ -0,0 +1,22 @@ +from cassandra.cluster import Cluster +import os + +def db_teardown(config): + print(" == DB TEARDOWN SCRIPT == ") + + print(f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}") + cluster = Cluster(config['cassandra_addresses'], port=config['cassandra_port']) + session = cluster.connect(config['cassandra_keyspace']) + print(f"Connection OK") + + for folder_name in os.listdir(config['setup_tables_dir']): + print(f"Dropping TABLE '{folder_name}'") + with open(f"{config['setup_tables_dir']}/{folder_name}/DROP.sql") as table_drop: + session.execute(table_drop.read()) + + with open(f"{config['setup_keyspace_dir']}/DROP.sql") as keyspace_drop: + print(f"Dropping KEYSPACE '{config['cassandra_keyspace']}'") + session.execute(keyspace_drop.read()) + + print("Done!") + cluster.shutdown() \ No newline at end of file diff --git a/setup/db/keyspace/CREATE.sql b/config/db/keyspace/CREATE.sql similarity index 52% rename from setup/db/keyspace/CREATE.sql rename to config/db/keyspace/CREATE.sql index 4ec3085..78114ee 100644 --- a/setup/db/keyspace/CREATE.sql +++ b/config/db/keyspace/CREATE.sql @@ -1 +1 @@ -CREATE KEYSPACE IF NOT EXISTS distributedunionfind WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 3}; \ No newline at end of file +CREATE KEYSPACE IF NOT EXISTS distributedunionfind WITH replication = {'class':'SimpleStrategy', 'replication_factor': 1}; \ No newline at end of file diff --git a/config/db/keyspace/DROP.sql b/config/db/keyspace/DROP.sql new file mode 100644 index 0000000..a52e6fb --- /dev/null +++ b/config/db/keyspace/DROP.sql @@ -0,0 +1 @@ +DROP KEYSPACE IF EXISTS distributedunionfind; \ No newline at end of file diff --git a/setup/db/keyspace/USE.sql b/config/db/keyspace/USE.sql similarity index 100% rename from setup/db/keyspace/USE.sql rename to config/db/keyspace/USE.sql diff --git a/config/db/tables/clusters/CREATE.sql b/config/db/tables/clusters/CREATE.sql new file mode 100644 index 0000000..43c3cfc --- /dev/null +++ b/config/db/tables/clusters/CREATE.sql @@ -0,0 +1,5 @@ +CREATE TABLE clusters( + address TEXT, + parent TEXT, + PRIMARY KEY (address) +); \ No newline at end of file diff --git a/config/db/tables/clusters/DROP.sql b/config/db/tables/clusters/DROP.sql new file mode 100644 index 0000000..202ede6 --- /dev/null +++ b/config/db/tables/clusters/DROP.sql @@ -0,0 +1 @@ +DROP TABLE clusters; \ No newline at end of file diff --git a/setup/db/tables/transactions/CREATE.sql b/config/db/tables/transactions/CREATE.sql similarity index 100% rename from setup/db/tables/transactions/CREATE.sql rename to config/db/tables/transactions/CREATE.sql diff --git a/config/db/tables/transactions/DROP.sql b/config/db/tables/transactions/DROP.sql new file mode 100644 index 0000000..27f9e9c --- /dev/null +++ b/config/db/tables/transactions/DROP.sql @@ -0,0 +1 @@ +DROP TABLE transactions; \ No newline at end of file diff --git a/scratchpad.py b/scratchpad.py new file mode 100644 index 0000000..225b77d --- /dev/null +++ b/scratchpad.py @@ -0,0 +1,32 @@ +import sys +import json +from cassandra.cluster import Cluster + +sys.path.append("config/db") + +config = json.load(open("./settings.json")) + +print( + f"Attempting Cassandra connection @ {config['cassandra_addresses']}:{config['cassandra_port']}") +cluster = Cluster(config['cassandra_addresses'], + port=config['cassandra_port']) +session = cluster.connect(config['cassandra_keyspace']) +print(f"Connection OK") + +result = session.execute("SELECT * FROM clusters") +print(result.all()) + + +""" +sc = pyspark.SparkContext('spark://osboxes:7077') + +data = sc.parallelize(list("aaa bbb cc dd e f")) +counts = data \ + .map(lambda x: (x, 1)) \ + .reduceByKey(add) \ + .sortBy(lambda x: x[1], ascending=False) \ + .collect() + +for (word, count) in counts: + print("{}: {}".format(word, count)) +""" \ No newline at end of file diff --git a/settings.json b/settings.json new file mode 100644 index 0000000..0732a80 --- /dev/null +++ b/settings.json @@ -0,0 +1,13 @@ +{ + "cassandra_addresses": ["127.0.0.1"], + "cassandra_port": 9042, + "cassandra_keyspace": "distributedunionfind", + + "setup_db_dir": "config/db", + "setup_tables_dir": "config/db/tables", + "setup_keyspace_dir": "config/db/keyspace", + + "tx_table_name": "transactions", + "clusters_table_name": "clusters" + +} \ No newline at end of file diff --git a/settings.py b/settings.py deleted file mode 100644 index 808b9e1..0000000 --- a/settings.py +++ /dev/null @@ -1,6 +0,0 @@ -cassandra_addresses = ['127.0.0.1'] -cassandra_port = 9042 -cassandra_keyspace = 'distributedunionfind' - -setup_db_dir = 'setup/db' -setup_tables_dir = setup_db_dir+"/tables" \ No newline at end of file diff --git a/setup.py b/setup.py index a0b592a..450b13a 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,12 @@ -from settings import cassandra_addresses, cassandra_port, setup_db_dir, setup_tables_dir -from cassandra.cluster import Cluster -import os +import sys +import json -print(" == DB SETUP SCRIPT == ") +sys.path.append("config/db") -print(f"Attempting Cassandra connection @ {cassandra_addresses}:{cassandra_port}") -cluster = Cluster(cassandra_addresses, port=cassandra_port) -session = cluster.connect() -print(f"Connection OK") +from db_setup import db_setup +from db_read_csv_txs import db_insert_csv_txs -with open(f"{setup_db_dir}/keyspace/CREATE.sql") as keyspace_create: - session.execute(keyspace_create.read()) +config = json.load(open("./settings.json")) -with open(f"{setup_db_dir}/keyspace/USE.sql") as keyspace_use: - session.execute(keyspace_use.read()) - -for folder_name in os.listdir(setup_tables_dir): - with open(f'{setup_tables_dir}/{folder_name}/CREATE.sql') as sql_create: - session.execute(sql_create.read()) - -session.execute("INSERT INTO transactions (tx_id,address,value,tx_hash,block_id,timestamp) VALUES(1697,'t1KmCvfPMgfQXeNosFqzAmvYdEoYfdnxnVA',15701,'18c23345908f5097456c5f0014411381fd9866790aa65b863aab24ee17453732',818,1477724947)") -res = session.execute('SELECT * FROM transactions') -print(res.one()) \ No newline at end of file +db_setup(config) +db_insert_csv_txs(config, "./small_test_data.csv") \ No newline at end of file diff --git a/setup/db/keyspace/DROP.sql b/setup/db/keyspace/DROP.sql deleted file mode 100644 index 19b4046..0000000 --- a/setup/db/keyspace/DROP.sql +++ /dev/null @@ -1 +0,0 @@ -DROP KEYSPACE IF EXISTS distributedunionfind \ No newline at end of file diff --git a/setup/db/tables/transactions/DROP.sql b/setup/db/tables/transactions/DROP.sql deleted file mode 100644 index bd4e7b7..0000000 --- a/setup/db/tables/transactions/DROP.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE transactions \ No newline at end of file diff --git a/small_test_data.csv b/small_test_data.csv new file mode 100644 index 0000000..a7ec08c --- /dev/null +++ b/small_test_data.csv @@ -0,0 +1,181 @@ +tx_id,address,value,tx_hash,block_id,timestamp +0,18XMcuNcPpUGLEgFogaXqZQWidNCyuWvrJ,4609,2179241ef47c59f043e8e5bfba33d4ccdcbad9bdfc245fab3f1ac0a0cf4bb95d,0,1477724947 +1,12yriKe117c24oqxoapTzWXwb2nPHbHVVi,442,bff57c7602bb55cfdfc07dd2e10274f4b82d6e0fb4eb311b3b06e74c38e61ff2,1,1477725342 +1,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2041,bff57c7602bb55cfdfc07dd2e10274f4b82d6e0fb4eb311b3b06e74c38e61ff2,1,1477725342 +2,1PzCy6TKe4GRn41WoQ9KQ4sfGL8kwSeAvj,3556,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635 +2,1FTkenQqmxtuhNATrV9evtkjfdo54L7hPN,3529,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635 +2,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,788,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635 +2,148Ebq2qHioHGFruAHvSHsut25L5F26TjG,1621,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635 +2,1FLV1Y8jem2G3Cxkh127sQnovaQvQCTtje,1915,859095d814516138ed8d9f66e7c7514ca0c452efeba668b193a64bbe1f985ddd,2,1477725635 +3,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4362,6305c45584d6f8e3daa1f4075eccf24757382641f402e1d8714286d1f49c1ffe,3,1477725636 +4,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,3028,c8f86fe8bdbaf231a03b395e4546e4c4e42ddbab3c2db479945592c0375a87b6,4,1477725729 +5,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2351,544a8690110099a4a1dc6831d6666199019e28cc1c7752ca965201d291c474ad,5,1477726050 +6,16SKLHfQdpF9DUzRFVpZ3RtDi1gViyHiby,1340,0cc5b12d22940ed0a596bf48ab84041c40ec5c35d8df32eea752857db296976f,6,1477726421 +6,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4668,0cc5b12d22940ed0a596bf48ab84041c40ec5c35d8df32eea752857db296976f,6,1477726421 +7,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4141,2b1fb25421601bd23d27e6b6d72331f713f44b6e7f15f2742f73322a31fc4f3a,7,1477726449 +8,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,422,3060c805f22cd47675d0cac8d4bd10839f6e9845cabe1e19c299e6cdee1b50a3,8,1477726589 +9,1K2KwATcG2vQ1L9s1qrLMci1oAMgTa1BYy,503,c0171e55ba1bdd2761e6acd80c7323255ee78d62d89d7dc95ac669b9d084e572,9,1477726862 +9,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,3737,c0171e55ba1bdd2761e6acd80c7323255ee78d62d89d7dc95ac669b9d084e572,9,1477726862 +10,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2981,7a93895f31104c5a3b21f0e32206d98f436fc5f652238e3965790efd80dc9c38,10,1477727261 +11,1Gri49ZrZ87JiD3yBXQZnYUdr8GV9z4MUH,1599,6c6cb74d5f0923bdee2b5e944dde20c8ecd5282e1708bb776a6cc847db2d531d,11,1477727495 +11,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4083,6c6cb74d5f0923bdee2b5e944dde20c8ecd5282e1708bb776a6cc847db2d531d,11,1477727495 +12,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4952,939a8f85fe523bb007898277e5e5e76ee673e252240cac5ad9f42cbefd4a3cf7,12,1477727563 +13,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,1824,a38382874101de402ea61a5a6b1d98ce370089d950df43d951f3c9dd53ed596b,13,1477727805 +14,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,1229,2da65b34c5c809743fe305b6819f19c7cc1f12137beafbc5836e4913a0184554,14,1477728074 +15,1Lzy6HQGL2TEpASEHSgmtvfut9auSkbmJs,3921,a9daae52ab4c501cef3bcf70e372cca192cfe4f0f9277f46f5ff4671e2a10de6,15,1477728463 +15,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2818,a9daae52ab4c501cef3bcf70e372cca192cfe4f0f9277f46f5ff4671e2a10de6,15,1477728463 +16,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2721,ecaea7121fedb14e90595a1bd7b54f8fd677c92266a3ebae361c3f8d8420aa4e,16,1477728645 +17,18hnWNqw8iqnkGg3h5592649XKQux7GUa6,3198,0d13804cd20bf131b7a9e104fc413a7e1b4e48fbfbdb99a017ee1ef6f573e8c7,17,1477728672 +17,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4447,0d13804cd20bf131b7a9e104fc413a7e1b4e48fbfbdb99a017ee1ef6f573e8c7,17,1477728672 +18,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,4226,09e2dabcf5a70bb0db27d0d1821c6fa5b1845ca3a02dd45285f7aae2459866c7,18,1477729019 +19,1MHViGVu72KWj7dMsqcypGVih385YoQzUb,2944,7bd825c8f456ff67511470d4ae996ae25e7badc7fe0a4b4c3866924062d4a0e0,19,1477729205 +20,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1665,5d70eee42a1e18cc17bc09e8d34754cca35d435b7d775cedffce9d0c1c7dbb0d,20,1477729336 +21,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,2386,ca80a3747917dc81c0f9a90de2185c66d71b704465b4de0aafa721f2763f387c,21,1477729702 +22,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3283,c5059fce2d1d3c2e8d7f31e5e4ce613ecc5db845fd47c8ca46b97dc266aa9dec,22,1477730093 +22,3Dm2pFiNk1PAjxHMuTrPUM8T5a1wxphAkP,1600,c5059fce2d1d3c2e8d7f31e5e4ce613ecc5db845fd47c8ca46b97dc266aa9dec,22,1477730093 +22,3QdE6N6dRdQNUVFy8Vv3twHuQRaa5kvu2U,1577,c5059fce2d1d3c2e8d7f31e5e4ce613ecc5db845fd47c8ca46b97dc266aa9dec,22,1477730093 +23,3QdE6N6dRdQNUVFy8Vv3twHuQRaa5kvu2U,2424,2276f51c7e33c5261fbc53104b743f15bbea2ad46722ac8592f118606ea944f1,23,1477730144 +24,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,368,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327 +24,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,233,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327 +24,3CZQg1dc1HJ41QJsxViuqxpJLXHGvWdy1w,1389,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327 +24,39TwnPh7pEr99ZyYfkUrwLkcTGmrVSBC3R,4526,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327 +24,3PcKS338Y2Je6uEZ77djmQJLChjdaLv75E,2779,98914ed1391f6408b91874dbd7f69f917d5bfc249d7593ec8e14de94747017d1,24,1477730327 +25,3CZQg1dc1HJ41QJsxViuqxpJLXHGvWdy1w,2848,b373d4a299942f628e68f7e7e84fdf4e3b5949f7592299e565a4d0e26bcec64d,25,1477730543 +26,3ESVZJmNGNDziUckU6qUDMmp6ue3v3ErfP,676,2808b24fd80ab192aa7afc55eb0bb31c33bacf1221aef543153ef677799c2ac6,26,1477730761 +26,3CZQg1dc1HJ41QJsxViuqxpJLXHGvWdy1w,2536,2808b24fd80ab192aa7afc55eb0bb31c33bacf1221aef543153ef677799c2ac6,26,1477730761 +27,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,3610,51adf572ffab1de3047f7bc46eff0a6708a62160aaef3cd579e3f8b2e8b9c2cf,27,1477730852 +28,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,688,ac63df67840aaf9a03f12b91a74961cff82609cec8357d8ab223d3153daa478a,28,1477730889 +29,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,3971,acd524b63d166835aeee515126e55670633bffc405ab387789e75c7dfb4fe3eb,29,1477731038 +30,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4928,1f642eff540fe878c3acb1518cd006f55797944b923643790222b85dd4ec3768,30,1477731296 +30,3EXdRFUCu5qBNtLwpWy8NvpBRdNPRsEvXQ,3307,1f642eff540fe878c3acb1518cd006f55797944b923643790222b85dd4ec3768,30,1477731296 +31,3H6sJfDuFPyAPrg8VaMKssd6jJ8eSQNN5C,1653,d3d5711d8b92437837db82331a4132e658922381afef5f60cc6b415ce1999e2d,31,1477731611 +32,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1486,4e690c8576cf921bac42befbfd55f2dcac5f6ea6763ab9b22deca052e15cbbb7,32,1477731703 +33,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1824,cdd555b0512bf1b23828de121ff22b1fc96b81c125c0fd78c51a75b787fcfff2,33,1477731720 +33,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,2108,cdd555b0512bf1b23828de121ff22b1fc96b81c125c0fd78c51a75b787fcfff2,33,1477731720 +34,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1362,e937af274b25f12cca28cb7bd27dab1621a2ca3d9907481cd56b4f942562bf78,34,1477732036 +35,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2341,a65c8de4f5d20706bd4109ad2662a0b36f022c52ede0004bb5fec131fb5cee98,35,1477732212 +35,34oeKULDnRqsiTToVXzYxYLwAuyi1QhTGD,837,a65c8de4f5d20706bd4109ad2662a0b36f022c52ede0004bb5fec131fb5cee98,35,1477732212 +35,32Yh4hKDiKP2h5jRX57HDSNsTJtpryCB8p,3671,a65c8de4f5d20706bd4109ad2662a0b36f022c52ede0004bb5fec131fb5cee98,35,1477732212 +36,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4933,127fa2027adf318c4ed8251b9a8310b20a00cbf83c39dc9ead67fc3561be5b43,36,1477732568 +37,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,1058,f1b9523ddc5574dbe98bfb6f897d3ff030b8873dd877bc230dc491669323618f,37,1477732824 +38,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,679,3514cece33629552536dc2c85fef8f9001ce64cceef3d00d8579458382dbe7e6,38,1477732916 +38,3LyMyw5q3VDmdc5y2uK52SxTFV6YtDRjEU,4161,3514cece33629552536dc2c85fef8f9001ce64cceef3d00d8579458382dbe7e6,38,1477732916 +39,3PCDL1umA9hXMp2YRDurFkQ8tMEjePyQyq,4510,a2e069ab489c5ed32cfbbfbf8c41f4e81d516991666338b1489dec404e1b5219,39,1477733228 +39,338DY888RLprEK6Sj92Ro1X4jspdF9WMTK,2349,a2e069ab489c5ed32cfbbfbf8c41f4e81d516991666338b1489dec404e1b5219,39,1477733228 +39,3FAH22bWvGHbD5xwdAviPD2NkypCcNtZfz,533,a2e069ab489c5ed32cfbbfbf8c41f4e81d516991666338b1489dec404e1b5219,39,1477733228 +39,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1250,a2e069ab489c5ed32cfbbfbf8c41f4e81d516991666338b1489dec404e1b5219,39,1477733228 +40,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,519,55143f25c59c95b9b66e1afb758589415a2e850d9ba3ffc5850ad5ffeeb248f4,40,1477733259 +41,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1022,12df97e178db9f13137eaf879c53f7df7464a67c69e6b418c09c8335af4d8a15,41,1477733269 +41,1NW3ovVf9DfpczHZ7YHajfXgjcN1BBxE8P,223,12df97e178db9f13137eaf879c53f7df7464a67c69e6b418c09c8335af4d8a15,41,1477733269 +42,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,1087,c6b9d1fdb7f8c20f4c578eebb2552b0adbbd1b28a957a8f56eca67de9ebdbb97,42,1477733283 +43,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,440,1b151c9fc241f6296b965c304dc3429786e57b0fb76c5cca351f60d3b531ebf9,43,1477733329 +44,3Gzq61vioahFFfyQR9A9nDt5UM61Jj9hJ9,2318,dee3d416d8beb94e9e170934fb29ad517cfef594fd078704da93ad74dfc9dad0,44,1477733382 +44,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4250,dee3d416d8beb94e9e170934fb29ad517cfef594fd078704da93ad74dfc9dad0,44,1477733382 +44,3GdURTjkyMgFnPjELgWTuVG1aLas3BhShA,3504,dee3d416d8beb94e9e170934fb29ad517cfef594fd078704da93ad74dfc9dad0,44,1477733382 +45,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4543,a4327bf59e02f3cebb63c25d585fca2ef723512a2a891a59890667f178fcc76c,45,1477733666 +46,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2831,6469cc94343c35bd3a9a316990f93c2f5a55806896aaf1be71b6cf3ba4f71295,46,1477733884 +47,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,1009,459054b60172dd81dd49930e805827c46b43bd68b8ea12dfc11a307183f58bf3,47,1477734056 +48,17P8Lsw4kFLhHLBTQwi9MCfR6xDF4MfH5F,4911,ded460c087bd6571a1cb0c4376258731a5769aaf56327ab77f418aebd55763f8,48,1477734388 +49,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,224,2d737e75a0cce777212d084d19450ac1396de684c81b48839246841e5cc05c84,49,1477734608 +50,1JNLN6LRiSrx5FWQ9dAtBgdPSo58TZREVc,514,17f64303dfba6f6d1b0c3faeeefaeaddb68c786742d449a1c2ddf8165582988b,50,1477734878 +50,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3338,17f64303dfba6f6d1b0c3faeeefaeaddb68c786742d449a1c2ddf8165582988b,50,1477734878 +50,1AjxAJKs12g16o5gx3YbfTJT197FJkq8Gd,2477,17f64303dfba6f6d1b0c3faeeefaeaddb68c786742d449a1c2ddf8165582988b,50,1477734878 +51,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,3960,99e53ce74a805ff8ce409fcb662b72c6783c2626a7d30f4a89c6675b9bee8281,51,1477735272 +52,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,2305,a05bb346e38e3906dee3387196bb0198319ac38d8a6c24ea9f159641c82ef549,52,1477735521 +53,19C6SDKcCXFZBPtcz5n857EjHKFJ9mSUAe,2904,76fca1eb2b263270dd412c75e7c83b1f3cc023ed9e60b55611ca6e8513f6a70b,53,1477735837 +53,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,4439,76fca1eb2b263270dd412c75e7c83b1f3cc023ed9e60b55611ca6e8513f6a70b,53,1477735837 +53,197uzNAN1bmRrQALTMWrh45HCA91LnAFDL,1650,76fca1eb2b263270dd412c75e7c83b1f3cc023ed9e60b55611ca6e8513f6a70b,53,1477735837 +54,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4407,b2d4e3a43827e1fd9279e2cefd9ce1df75b8da0c80cad131f4105c6f3c2e5e23,54,1477735861 +55,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,566,993544404fb7c7a12937c644c47f778670dd6ef3e57938809667f2794ca0f85d,55,1477735915 +56,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2603,ba75c0a7c4a5d1ef9b705f8309f415bc085354c9d8e2811d27919a31f33f7952,56,1477736274 +57,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2021,40680e7d122a5c24d0b14a5a6b3ad32322dd5aa904fccf6fc501fdaa0ae92c55,57,1477736499 +58,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2564,cf2fc800424cad5f6e6ca9e6fe867610769e6380f05cbf9d25c5af957a6565af,58,1477736684 +59,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3689,b87f98cd451fac55a912c27f02ef74401e8acd93419ca1d67d2b1a27eff7b42c,59,1477736774 +60,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,998,46591437f61e98a5c50146ba2166b41bebc2d6b73b12f2743ec1e3054fe79023,60,1477736878 +61,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4415,766970297f7cf1359ce38a146add3b8c4dc34ab9d9ee1913feb262cb2cfb41d1,61,1477736989 +62,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,991,4053ec2da3e43b45abf6f4001c6b88753e11818c172bf74d895679ebdce107bf,62,1477737035 +63,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3660,a39ef4617c802782d6e96fbb66d38f637e9937df810ea178262933e41cdbc5b8,63,1477737405 +64,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,815,e15a1524bfb8e476db962bb69870411cfad1154a071f1c06aaca0798da54dda3,64,1477737724 +65,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,817,36fc7f8881f09f7fa968bbde9d74cc0ceb9ea3d404ebc111694876bd2c863bb4,65,1477737939 +65,1JMgMQMK9JC9CBodvjxvLtoE42fgreGbHP,3649,36fc7f8881f09f7fa968bbde9d74cc0ceb9ea3d404ebc111694876bd2c863bb4,65,1477737939 +65,1QEGqswF6jcNmxG5Sejhf5puNpaQ27HBDX,4661,36fc7f8881f09f7fa968bbde9d74cc0ceb9ea3d404ebc111694876bd2c863bb4,65,1477737939 +66,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,4886,c4b4b659cb94522fb2143c04d4a6fa4414b2e5aac6594f3e5251585e50332afa,66,1477738165 +67,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3780,763f8c9b311c0f2ba49cc4f20ee70704445165de6efaa371aeccc0a756cb6a90,67,1477738196 +67,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,4369,763f8c9b311c0f2ba49cc4f20ee70704445165de6efaa371aeccc0a756cb6a90,67,1477738196 +68,1PmCGRS6Crrfz2QcmT3XUGcXtvLj2cxUJR,4157,6989ef40065760ff1318c49c1447d1b8f1da9c2c729f895b91215c2851e31b2c,68,1477738542 +68,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,3383,6989ef40065760ff1318c49c1447d1b8f1da9c2c729f895b91215c2851e31b2c,68,1477738542 +69,13S8ipxJvzz22xKCrHfe3MWC6HTDR8gYhS,1739,58b1c24524e5fd629d29389aff989bcc90c2533c7e1401c065faed12ff3a5233,69,1477738636 +69,1NEkBXK77FNciK2BwqA55diDUhgsvjyh5V,768,58b1c24524e5fd629d29389aff989bcc90c2533c7e1401c065faed12ff3a5233,69,1477738636 +69,1F8frjsrDpNuANKuhMxz1pTrt3ThNAgeKF,2987,58b1c24524e5fd629d29389aff989bcc90c2533c7e1401c065faed12ff3a5233,69,1477738636 +70,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2281,52b2efe12c32076f18830b70d228bfa5e56bfd7119546c94779ec9a16f32bf68,70,1477739005 +71,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2551,8fe146a5e1a141e9b2bb0dbf2aa0a25a2fb00974a47b9c5ed8a78f671c64c9e4,71,1477739028 +71,1EksR7DeEodWTBwoPmzbmfFYt1fAMzD2RU,2613,8fe146a5e1a141e9b2bb0dbf2aa0a25a2fb00974a47b9c5ed8a78f671c64c9e4,71,1477739028 +71,17pL4RA7GwV87mjLQRuy4URUpBRhBpAUfx,1461,8fe146a5e1a141e9b2bb0dbf2aa0a25a2fb00974a47b9c5ed8a78f671c64c9e4,71,1477739028 +71,15YK8ZZyL7YezuwpraQodjF56pws7TRjd2,70,8fe146a5e1a141e9b2bb0dbf2aa0a25a2fb00974a47b9c5ed8a78f671c64c9e4,71,1477739028 +72,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3595,e2364f281a207b154678561f5c29f2bea911940445b4446f3bd3e98be2b80719,72,1477739078 +73,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,3634,a646b1b87b24916cdf6ce439d590cf4786c33a877ef7442aa379b05fe938edf0,73,1477739116 +74,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,604,8bd3f57789e48effe34db65627ae5759c7be8b3290ab16e89e0711d3591c8be9,74,1477739120 +75,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,4347,d48bb09aa5b8b85df707be1d8de322d3a282353a694335745befc76c6bcab09e,75,1477739348 +76,1JhZvzBCVLkbEMvs8xhy34RpfwsriTJUrt,2597,80927bf38ad2a17c63f8bcad914696adc1146d822a4fd9abaec4812f0102de41,76,1477739562 +77,1Gtcs8i8SkQNWAwxi1WzLFPWmt7JT72n7n,352,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797 +77,1BBeBwTNJmWRFcmfpT36Xzff19mc65Ncnu,3039,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797 +77,1AAWJ3CKn2eaM6PHffukNunPmARB7kDsqU,3605,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797 +77,16fdPvFnSFYpQEWqPXG3U5rmm78YZbiuXp,2792,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797 +77,17aoiN8dc8sk55EKFQtTQ31XhJHAbHWrG,4116,5aec509f9d06a77c3747e2afd8e84ab5c34f1897aef314dff61287d58c1b1323,77,1477739797 +78,1JheBVxaY5ZyWMgDBe4KdG2DPQrS1sVvKv,3862,58f4dfdea4d9fb9204965469a7d20a1feba6dcf104200bb24bd4f50d9baab5df,78,1477739918 +78,1Fkt2RPovzEUMf2118RJUTRgJYKeEgFwFA,460,58f4dfdea4d9fb9204965469a7d20a1feba6dcf104200bb24bd4f50d9baab5df,78,1477739918 +78,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,1223,58f4dfdea4d9fb9204965469a7d20a1feba6dcf104200bb24bd4f50d9baab5df,78,1477739918 +79,1255XDfmdQTfURAiKGz83bdstfFNyt6vUN,4825,46057443f11993dd2a979548b9fb82329b25fc4cb73500b479e06c8f18712e1e,79,1477740069 +80,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,3755,6fbff7e292b2040cdc7d7dbffa56054f58e7c1bbb64d226c85b2316a50835b90,80,1477740305 +81,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,2656,18cd57e874fc6e8b13a68d6f18c761c0906ca0bf26cdf70b9e88fee405d6afdc,81,1477740420 +82,1Fkt2RPovzEUMf2118RJUTRgJYKeEgFwFA,577,7980fabd136a2f7edd31c7be3e374761efc79e130a656d018706af02af76b04d,82,1477740456 +83,1NoXX6QyKRw3cPnbMC4LP9qiGZdXYcsWR6,3531,60b605e46b3415484377a2e6c748436b15b34385ae4cbc562e38046c9eda296f,83,1477740489 +83,1255XDfmdQTfURAiKGz83bdstfFNyt6vUN,138,60b605e46b3415484377a2e6c748436b15b34385ae4cbc562e38046c9eda296f,83,1477740489 +84,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,2486,323b60dae16e1639d3d97eec2eaeb743d552b7ece6c336999dd3f1b8bbb7a1f6,84,1477740669 +85,1AAWJ3CKn2eaM6PHffukNunPmARB7kDsqU,2287,5285bf90f158d58f4d09a7f42030c7fd7aae557a8463d9dc41d39eda783500f8,85,1477740882 +86,1NoXX6QyKRw3cPnbMC4LP9qiGZdXYcsWR6,2064,64e641282b4cbf0155740c20e36242aec85fef27f6407220a58d2496e4556df4,86,1477741104 +87,1341WZDtruCsQDthA6scJLSkRtojwBAuJu,2743,58ef204b34791846327fe625c8c18a35cd433de40614e494a066b1c02918f1cb,87,1477741388 +88,1JDjYwc8EJViauMFoSt8KZLaWscCW2huKu,2590,fb17f512882b9ca4f78901139947a4586ab45efff93f3c0e62e30275eb0a7046,88,1477741413 +88,1Gtcs8i8SkQNWAwxi1WzLFPWmt7JT72n7n,1209,fb17f512882b9ca4f78901139947a4586ab45efff93f3c0e62e30275eb0a7046,88,1477741413 +88,1BBeBwTNJmWRFcmfpT36Xzff19mc65Ncnu,3320,fb17f512882b9ca4f78901139947a4586ab45efff93f3c0e62e30275eb0a7046,88,1477741413 +88,1EPKRbWGii8rKfBM3weGXx86mSeZBSdc9,1669,fb17f512882b9ca4f78901139947a4586ab45efff93f3c0e62e30275eb0a7046,88,1477741413 +89,1MyujkWgHoqMRxiG7aCrLUTdKgZ9eR1pj4,2914,ff9413176e9a9baa6fb1b6e4548e8799c190b2296ca883b3a4ab98e0aeeb59cf,89,1477741755 +90,1DaVpKn8WyNFspk9eYn7THcJhKwSnAkY2P,3888,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1341WZDtruCsQDthA6scJLSkRtojwBAuJu,2687,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1Gtcs8i8SkQNWAwxi1WzLFPWmt7JT72n7n,51,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1Ai6zMeNeRvKYgkzuMDcYVhn9ounzzvrJM,87,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,18o3Zxu3wZ2udocLDprnDYbjfCj2vfAPVb,256,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1FKkzWTUToYWSrBntn2bNJCWCySVFmmW47,3816,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1973XU2xwYg8mwH6q111zdn2pEEzBSS66F,4081,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1ANYUUEX8HNPEN59RyhfYG3wLbr5zTLET3,2198,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1DUpoFzu85KRWZAVg4YHkeHLPsp5eEWo5E,824,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1MgU7ZSZEacoBsk9FaDnNrr8iLHDCdW3zy,181,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1NoXX6QyKRw3cPnbMC4LP9qiGZdXYcsWR6,3255,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1MyujkWgHoqMRxiG7aCrLUTdKgZ9eR1pj4,2482,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,13LsCQgxkwV1TBagv2Dhd1P93KFgTJjCtL,3502,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1Nk2nNSxY654swupCfWF8XfoJq9pktbChB,725,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +90,1Fv9wFQ7zHPhDykeo3KoLRxWaxwm6r4PZ1,2658,6d531da5b7233c39b036af4c8f697a3bb388290486636f4820bb1324dfcf67d2,90,1477742009 +91,1ANYUUEX8HNPEN59RyhfYG3wLbr5zTLET3,2722,bef4df40757a1bfb0a6ae91a2b7b15eb30aecda80d557722064518b914a1578a,91,1477742144 +92,1MU36WKxBu45TcAfkVpJATwwHbbSnimbMz,4214,63c8994839b5bef056b655baebb7141d6b569de622d29dcb33d6e855a9cd1a15,92,1477742239 +92,1Fv9wFQ7zHPhDykeo3KoLRxWaxwm6r4PZ1,384,63c8994839b5bef056b655baebb7141d6b569de622d29dcb33d6e855a9cd1a15,92,1477742239 +93,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,2808,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278 +93,1UGpcdT7JwCYVfKcHyvdNEKjSkbdZs3MK,3406,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278 +93,1EzWmRUxdJ2KRddYiMCqCYXQa79V4SVYtK,1584,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278 +93,16KfsnZJqRhUBM4zJigYzDMTrpBCJnFFZc,89,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278 +93,17D77wQF9gQuoX7muR9Mk1xjLBEEmoNrhK,3317,36ea82e071dc00e73355fe92ae2e433fd7ba174b2a335634468dd8bfcef968a0,93,1477742278 +94,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,2659,30f07a8db3da1f64b676dd82dfded1e2f8416815ca2d98a2a0bdd7da23af8c80,94,1477742635 +95,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,1698,7dfdb42e4f84e5bb2a497fd28020479b1dabd28a985344cc5f577bd9f7c678d4,95,1477743011 +96,1hJz3Dc2f7cPvBspdiY5p1UMsKMA2gogT,3935,d15196c305dd77f2d3e3259223cff1eaf490bffbca9d62ce03abbb08bcf6fd33,96,1477743384 +96,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,4052,d15196c305dd77f2d3e3259223cff1eaf490bffbca9d62ce03abbb08bcf6fd33,96,1477743384 +96,19wz95smVq3AWCcpst8b8CaPT5m2a2Sf2U,3200,d15196c305dd77f2d3e3259223cff1eaf490bffbca9d62ce03abbb08bcf6fd33,96,1477743384 +97,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,4341,63271b55cd4bc253433b66bb559cd55258d44e0353a70c2fe5e13c81f907eff0,97,1477743664 +97,1DS7SAKyEynj8oH5s9PgxE2BUfbNckd355,4525,63271b55cd4bc253433b66bb559cd55258d44e0353a70c2fe5e13c81f907eff0,97,1477743664 +98,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,4786,f82f5d6cc0b8770b8ece9b7fad661412994999921f17a1d0446504ddca87851a,98,1477743799 +99,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,874,870382a095c8364d1da768c6787a44ec3cc15b6fadd5085d85ed8c813cb5da5a,99,1477743873 +99,1BwcTvisQ8TFnVMtva78zWMyvqqr7k4dyV,152,870382a095c8364d1da768c6787a44ec3cc15b6fadd5085d85ed8c813cb5da5a,99,1477743873 +99,1H99UihszQWoppVqWvp5LDjR8PUTFQ58EU,576,870382a095c8364d1da768c6787a44ec3cc15b6fadd5085d85ed8c813cb5da5a,99,1477743873 +100,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,670,d08a6ec49f285c361a63d1c9528b47ef4d2efb3a41a9f6fbf17078dd44c3483f,100,1477743908 +101,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,469,3665474c37ebb5524a3bcceb8249b6c84e1b7fcefe70f1c9db898673a6c84c2e,101,1477744047 +102,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,3200,3134370264311c339b8fb8e1676bf053cec6f18100f5941c2ae4b7ec9cbd3ce8,102,1477744286 +103,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,3538,e80d703ccb172d18fb0a8f09112e540d1b44584410174bcad7a5c1e1658feddb,103,1477744536 +104,1F4yNuWhNXkrf4pE4QMfZLrt7mRVzw9rrK,4073,ef3e9380f2b2eb6693fdda09a5ec9f4329d7b48e91e5f9585b2eee09227fff58,104,1477744928 \ No newline at end of file diff --git a/src/spark/main.py b/src/spark/main.py new file mode 100644 index 0000000..c43f718 --- /dev/null +++ b/src/spark/main.py @@ -0,0 +1,76 @@ +from gc import collect +from sqlite3 import Row +from typing import Iterable +from operator import add + +from pyspark.sql import SparkSession +from pyspark.sql import functions as F + + + +spark = SparkSession.builder \ + .appName('SparkCassandraApp') \ + .config('spark.cassandra.connection.host', 'localhost') \ + .config('spark.cassandra.connection.port', '9042') \ + .config('spark.cassandra.output.consistency.level', 'ONE') \ + .config("spark.sql.extensions", "com.datastax.spark.connector.CassandraSparkExtensions") \ + .config('directJoinSetting', 'on') \ + .master('spark://osboxes:7077') \ + .getOrCreate() + +spark.conf.set("spark.sql.catalog.myCatalog", + "com.datastax.spark.connector.datasource.CassandraCatalog") + + +tx_addr_groups = spark.read.table("myCatalog.distributedunionfind.transactions") \ + .groupBy("tx_id") \ + .agg(F.collect_set('address').alias('addresses')) \ + .toLocalIterator() + +def insertCluster (row): + addrs: Iterable[str] = row['addresses'] + df = spark.createDataFrame(map(lambda addr: (addr, addrs[0]), addrs), schema=['address', 'parent']) + + df.writeTo("myCatalog.distributedunionfind.clusters").overwrite() + +""" + tuple structure: + Row => Row(parent=addr, addresses=list[addr] + Iterable[str] => list[addr] +""" +def find(data: tuple[Row, Iterable[str]]): + cluster = data[0] + tx = data[1] + + clusteraddresses = cluster['addresses'] + [cluster['parent']] + + if any(x in tx for x in clusteraddresses): + return cluster['parent'] + else: + return None + +for addr_group in tx_addr_groups: + clusters_df = spark.read.table("myCatalog.distributedunionfind.clusters") + + clusters = clusters_df \ + .groupBy("parent") \ + .agg(F.collect_set('address').alias('addresses')) + + if (clusters.count() == 0): + insertCluster(addr_group) + continue + + df = clusters.rdd \ + .map(lambda cluster: (cluster, addr_group['addresses'])) \ + .map(find) \ + .filter(lambda x: x != None) \ + .collect() + + if(len(df) == 0): + insertCluster(addr_group) + continue + + print(addr_group) + print(df) + + break diff --git a/start_services.sh b/start_services.sh new file mode 100755 index 0000000..992d294 --- /dev/null +++ b/start_services.sh @@ -0,0 +1,6 @@ +SPARK_HOME="/home/osboxes/Downloads/spark-3.2.2-bin-hadoop3.2" +SPARK_MASTER="spark://osboxes:7077" + +"$SPARK_HOME"/sbin/start-master.sh +SPARK_WORKER_INSTANCES=5 "$SPARK_HOME"/sbin/start-worker.sh "$SPARK_MASTER" +docker run -d -p 9042:9042 cassandra \ No newline at end of file diff --git a/submit.sh b/submit.sh new file mode 100755 index 0000000..37528d5 --- /dev/null +++ b/submit.sh @@ -0,0 +1,14 @@ +SPARK_HOME="/home/osboxes/Downloads/spark-3.2.2-bin-hadoop3.2" +MEMORY="4g" +SPARK_MASTER="spark://osboxes:7077" +CASSANDRA_HOST="localhost" + + +"$SPARK_HOME"/bin/spark-submit \ +--master "$SPARK_MASTER" \ +--conf spark.executor.memory="$MEMORY" \ +--conf spark.cassandra.connection.host="$CASSANDRA_HOST" \ +--conf spark.sql.session.timeZone=UTC \ +--conf spark.sql.extensions=com.datastax.spark.connector.CassandraSparkExtensions \ +--packages com.datastax.spark:spark-cassandra-connector_2.12:3.2.0 \ +./src/spark/main.py diff --git a/teardown.py b/teardown.py deleted file mode 100644 index 6611424..0000000 --- a/teardown.py +++ /dev/null @@ -1,21 +0,0 @@ -from settings import cassandra_addresses, cassandra_port, cassandra_keyspace, setup_tables_dir, setup_db_dir -from cassandra.cluster import Cluster -import os - -print(" == DB TEARDOWN SCRIPT == ") - -print(f"Attempting Cassandra connection @ {cassandra_addresses}:{cassandra_port}") -cluster = Cluster(cassandra_addresses, port=cassandra_port) -session = cluster.connect(cassandra_keyspace) -print(f"Connection OK") - -for folder_name in os.listdir(setup_tables_dir): - print(f"Dropping table {folder_name}") - with open(f'{setup_tables_dir}/{folder_name}/DROP.sql') as sql_drop: - session.execute(sql_drop.read()) - -with open(f"{setup_db_dir}/keyspace/DROP.sql") as keyspace_create: - print(f"Dropping keyspace {cassandra_keyspace}") - session.execute(keyspace_create.read()) - -print("Done!")