Skip to content
This repository has been archived by the owner on Jan 8, 2024. It is now read-only.

Commit

Permalink
Update data model for AddressTransactions in Spark
Browse files Browse the repository at this point in the history 
Remove unused field `timestamp` (not used in graphsense-rest).

See #41
  • Loading branch information
Rainer Stuetz committed Aug 16, 2021
1 parent d164727 commit 1199e4d
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 34 deletions.
3 changes: 1 addition & 2 deletions src/main/scala/info/graphsense/Model.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,7 @@ case class AddressTransaction(
addressId: Int,
txId: Long,
value: Long,
blockId: Int,
timestamp: Int
blockId: Int
)

case class BasicAddress(
Expand Down
8 changes: 2 additions & 6 deletions src/main/scala/info/graphsense/Transformation.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,6 @@ class Transformation(
}

def computeAddressTransactions(
tx: Dataset[Transaction],
regInputs: Dataset[RegularInput],
regOutputs: Dataset[RegularOutput],
addressIds: Dataset[AddressId]
Expand All @@ -305,11 +304,8 @@ class Transformation(
.withColumn(F.value, -col(F.value))
.union(regOutputs.drop(F.n))
.groupBy(F.txId, F.address)
.agg(sum(F.value).as(F.value))
.join(
tx.select(F.txId, F.blockId, F.timestamp).distinct(),
F.txId
)
// blockId is constant in each group, get single value with min
.agg(sum(F.value).as(F.value), min(F.blockId).as(F.blockId))
.join(addressIds, Seq(F.address))
.drop(F.addressPrefix, F.address)
.transform(t.withIdGroup(F.addressId, F.addressIdGroup))
Expand Down
1 change: 0 additions & 1 deletion src/main/scala/info/graphsense/TransformationJob.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,6 @@ object TransformationJob {
val addressTransactions =
transformation
.computeAddressTransactions(
transactions,
regInputs,
regOutputs,
addressIds
Expand Down
48 changes: 24 additions & 24 deletions src/test/resources/reference/address_txs.json
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
{"value": 2500000, "blockId": 1, "txId": 1, "timestamp": 1230948000, "addressIdGroup": 0, "addressId": 0}
{"value": -2500000, "blockId": 2, "txId": 3, "timestamp": 1231466400, "addressIdGroup": 0, "addressId": 0}
{"value": 2500000, "blockId": 2, "txId": 2, "timestamp": 1231466400, "addressIdGroup": 0, "addressId": 1}
{"value": -2500000, "blockId": 3, "txId": 5, "timestamp": 1231552800, "addressIdGroup": 0, "addressId": 1}
{"value": 1480000, "blockId": 2, "txId": 3, "timestamp": 1231466400, "addressIdGroup": 1, "addressId": 2}
{"value": -1480000, "blockId": 3, "txId": 5, "timestamp": 1231552800, "addressIdGroup": 1, "addressId": 2}
{"value": 495000, "blockId": 2, "txId": 3, "timestamp": 1231466400, "addressIdGroup": 1, "addressId": 3}
{"value": -150000, "blockId": 3, "txId": 6, "timestamp": 1231552800, "addressIdGroup": 1, "addressId": 3}
{"value": -345000, "blockId": 4, "txId": 9, "timestamp": 1231639200, "addressIdGroup": 1, "addressId": 3}
{"value": 495000, "blockId": 2, "txId": 3, "timestamp": 1231466400, "addressIdGroup": 2, "addressId": 4}
{"value": -495000, "blockId": 4, "txId": 8, "timestamp": 1231639200, "addressIdGroup": 2, "addressId": 4}
{"value": 2500000, "blockId": 3, "txId": 4, "timestamp": 1231552800, "addressIdGroup": 2, "addressId": 5}
{"value": -2500000, "blockId": 4, "txId": 8, "timestamp": 1231639200, "addressIdGroup": 2, "addressId": 5}
{"value": 1325000, "blockId": 3, "txId": 5, "timestamp": 1231552800, "addressIdGroup": 3, "addressId": 6}
{"value": -1325000, "blockId": 4, "txId": 10, "timestamp": 1231639200, "addressIdGroup": 3, "addressId": 6}
{"value": 100000, "blockId": 3, "txId": 5, "timestamp": 1231552800, "addressIdGroup": 3, "addressId": 7}
{"value": -100000, "blockId": 4, "txId": 10, "timestamp": 1231639200, "addressIdGroup": 3, "addressId": 7}
{"value": 2350000, "blockId": 3, "txId": 5, "timestamp": 1231552800, "addressIdGroup": 4, "addressId": 8}
{"value": 140000, "blockId": 3, "txId": 6, "timestamp": 1231552800, "addressIdGroup": 4, "addressId": 9}
{"value": 340000, "blockId": 4, "txId": 9, "timestamp": 1231639200, "addressIdGroup": 4, "addressId": 9}
{"value": 2500000, "blockId": 4, "txId": 7, "timestamp": 1231639200, "addressIdGroup": 5, "addressId": 10}
{"value": 4300000, "blockId": 4, "txId": 8, "timestamp": 1231639200, "addressIdGroup": 5, "addressId": 11}
{"value": 149000, "blockId": 4, "txId": 8, "timestamp": 1231639200, "addressIdGroup": 6, "addressId": 12}
{"value": 1400000, "blockId": 4, "txId": 10, "timestamp": 1231639200, "addressIdGroup": 6, "addressId": 13}
{"value": 2500000, "blockId": 1, "txId": 1, "addressIdGroup": 0, "addressId": 0}
{"value": -2500000, "blockId": 2, "txId": 3, "addressIdGroup": 0, "addressId": 0}
{"value": 2500000, "blockId": 2, "txId": 2, "addressIdGroup": 0, "addressId": 1}
{"value": -2500000, "blockId": 3, "txId": 5, "addressIdGroup": 0, "addressId": 1}
{"value": 1480000, "blockId": 2, "txId": 3, "addressIdGroup": 1, "addressId": 2}
{"value": -1480000, "blockId": 3, "txId": 5, "addressIdGroup": 1, "addressId": 2}
{"value": 495000, "blockId": 2, "txId": 3, "addressIdGroup": 1, "addressId": 3}
{"value": -150000, "blockId": 3, "txId": 6, "addressIdGroup": 1, "addressId": 3}
{"value": -345000, "blockId": 4, "txId": 9, "addressIdGroup": 1, "addressId": 3}
{"value": 495000, "blockId": 2, "txId": 3, "addressIdGroup": 2, "addressId": 4}
{"value": -495000, "blockId": 4, "txId": 8, "addressIdGroup": 2, "addressId": 4}
{"value": 2500000, "blockId": 3, "txId": 4, "addressIdGroup": 2, "addressId": 5}
{"value": -2500000, "blockId": 4, "txId": 8, "addressIdGroup": 2, "addressId": 5}
{"value": 1325000, "blockId": 3, "txId": 5, "addressIdGroup": 3, "addressId": 6}
{"value": -1325000, "blockId": 4, "txId": 10, "addressIdGroup": 3, "addressId": 6}
{"value": 100000, "blockId": 3, "txId": 5, "addressIdGroup": 3, "addressId": 7}
{"value": -100000, "blockId": 4, "txId": 10, "addressIdGroup": 3, "addressId": 7}
{"value": 2350000, "blockId": 3, "txId": 5, "addressIdGroup": 4, "addressId": 8}
{"value": 140000, "blockId": 3, "txId": 6, "addressIdGroup": 4, "addressId": 9}
{"value": 340000, "blockId": 4, "txId": 9, "addressIdGroup": 4, "addressId": 9}
{"value": 2500000, "blockId": 4, "txId": 7, "addressIdGroup": 5, "addressId": 10}
{"value": 4300000, "blockId": 4, "txId": 8, "addressIdGroup": 5, "addressId": 11}
{"value": 149000, "blockId": 4, "txId": 8, "addressIdGroup": 6, "addressId": 12}
{"value": 1400000, "blockId": 4, "txId": 10, "addressIdGroup": 6, "addressId": 13}
1 change: 0 additions & 1 deletion src/test/scala/info/graphsense/TransformationTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,6 @@ class TransformationTest

val addressTransactions =
t.computeAddressTransactions(
transactions,
regInputs,
regOutputs,
addressIds
Expand Down

0 comments on commit 1199e4d

Please sign in to comment.