From b17954adc29df3fe14839695a3c6d4ca7d60b188 Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Mon, 14 Dec 2015 23:14:52 +0200 Subject: [PATCH 01/12] Split interface into BulletinBoardClient and AsyncBulletinBoardClient. Added Batch Messages Bulletin Board Client interface and associated ProtoBufs. Returned simple implementation of BulletinBoardClient. Made ThreadedBulletinBoardClient extend SimpleBulletinBoardClient. Fixed an issue in SQLite where identical Signatures could be added to the same message. --- .../SimpleBulletinBoardClient.java | 32 +++++----- .../ThreadedBulletinBoardClient.java | 45 +++++++------- .../BulletinBoardClientIntegrationTest.java | 3 +- .../sqlserver/SQLiteQueryProvider.java | 3 +- .../AsyncBulletinBoardClient.java | 62 +++++++++++++++++++ .../bulletinboard/BulletinBoardClient.java | 23 ++++--- .../meerkat/bulletinboard/SignedBatch.java | 50 +++++++++++++++ .../main/proto/meerkat/BulletinBoardAPI.proto | 27 ++++++++ 8 files changed, 196 insertions(+), 49 deletions(-) create mode 100644 meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java create mode 100644 meerkat-common/src/main/java/meerkat/bulletinboard/SignedBatch.java diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java index f340cae..4244f15 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java @@ -5,8 +5,7 @@ import meerkat.comm.CommunicationException; import meerkat.crypto.Digest; import meerkat.crypto.concrete.SHA256Digest; import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.protobuf.Voting; -import meerkat.protobuf.Voting.BulletinBoardClientParams; +import meerkat.protobuf.Voting.*; import meerkat.rest.*; import java.util.List; @@ -19,23 +18,24 @@ import javax.ws.rs.core.Response; /** * Created by Arbel Deutsch Peled on 05-Dec-15. + * Implements BulletinBoardClient interface in a simple, straightforward manner */ -public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { +public class SimpleBulletinBoardClient implements BulletinBoardClient{ - private List meerkatDBs; + protected List meerkatDBs; - private Client client; + protected Client client; - private Digest digest; + protected Digest digest; /** * Stores database locations and initializes the web Client * @param clientParams contains the data needed to access the DBs */ -// @Override - public void init(Voting.BulletinBoardClientParams clientParams) { + @Override + public void init(BulletinBoardClientParams clientParams) { - meerkatDBs = clientParams.getBulletinBoardAddressList(); + this.meerkatDBs = clientParams.getBulletinBoardAddressList(); client = ClientBuilder.newClient(); client.register(ProtobufMessageBodyReader.class); @@ -52,7 +52,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { * @return the message ID for later retrieval * @throws CommunicationException */ -// @Override + @Override public MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException { WebTarget webTarget; @@ -88,7 +88,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { * @param id is the requested message ID * @return the number of DBs in which retrieval was successful */ -// @Override + @Override public float getRedundancy(MessageID id) { WebTarget webTarget; Response response; @@ -125,7 +125,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { * @param filterList return only messages that match the filters (null means no filtering). * @return */ -// @Override + @Override public List readMessages(MessageFilterList filterList) { WebTarget webTarget; Response response; @@ -154,8 +154,8 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { return null; } -// @Override -// public void registerNewMessageCallback(MessageCallback callback, MessageFilterList filterList) { -// callback.handleNewMessage(readMessages(filterList)); -// } + public void close() { + client.close(); + } + } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java index bb46c32..dd103c6 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java @@ -6,10 +6,8 @@ import meerkat.bulletinboard.callbacks.GetRedundancyFutureCallback; import meerkat.bulletinboard.callbacks.PostMessageFutureCallback; import meerkat.bulletinboard.callbacks.ReadMessagesFutureCallback; import meerkat.comm.CommunicationException; -import meerkat.crypto.Digest; -import meerkat.crypto.concrete.SHA256Digest; import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.protobuf.Voting; +import meerkat.protobuf.Voting.*; import java.util.List; import java.util.concurrent.Executors; @@ -17,22 +15,16 @@ import java.util.concurrent.TimeUnit; /** * Created by Arbel Deutsch Peled on 05-Dec-15. - * Thread-based implementation of a Bulletin Board Client. + * Thread-based implementation of a Async Bulletin Board Client. * Features: * 1. Handles tasks concurrently. * 2. Retries submitting */ -public class ThreadedBulletinBoardClient implements BulletinBoardClient { +public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient implements AsyncBulletinBoardClient { private final static int THREAD_NUM = 10; ListeningExecutorService listeningExecutor; - private Digest digest; - - private List meerkatDBs; - private String postSubAddress; - private String readSubAddress; - private final static int READ_MESSAGES_RETRY_NUM = 1; private int minAbsoluteRedundancy; @@ -44,15 +36,13 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient { * @param clientParams contains the required information */ @Override - public void init(Voting.BulletinBoardClientParams clientParams) { + public void init(BulletinBoardClientParams clientParams) { - meerkatDBs = clientParams.getBulletinBoardAddressList(); + super.init(clientParams); - minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * meerkatDBs.size()); + minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * clientParams.getBulletinBoardAddressCount()); - listeningExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(THREAD_NUM)); - - digest = new SHA256Digest(); + listeningExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(THREAD_NUM)); } @@ -78,13 +68,21 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient { return MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build(); } + @Override + public MessageID postBatch(byte[] signerId, int batchId, List batchDataList, int startPosition, ClientCallback callback) { + return null; //TODO: Implement + } + + @Override + public MessageID postBatch(byte[] signerId, int batchId, List batchDataList, ClientCallback callback) { + return null; //TODO: Implement + } + /** * Access each database and search for a given message ID * Return the number of databases in which the message was found * Only try once per DB * Ignore communication exceptions in specific databases - * @param id is the requested message ID - * @return the number of DBs in which retrieval was successful */ @Override public void getRedundancy(MessageID id, ClientCallback callback) { @@ -101,8 +99,6 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient { * Go through the DBs and try to retrieve messages according to the specified filter * If at the operation is successful for some DB: return the results and stop iterating * If no operation is successful: return null (NOT blank list) - * @param filterList return only messages that match the filters (null means no filtering). - * @return */ @Override public void readMessages(MessageFilterList filterList, ClientCallback> callback) { @@ -116,8 +112,15 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient { } + @Override + public void readBatch(byte[] signerId, int batchId, ClientCallback callback) { + // TODO: Implement + } + @Override public void close() { + super.close(); + try { listeningExecutor.shutdown(); while (! listeningExecutor.isShutdown()) { diff --git a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java b/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java index dda76c7..c090c92 100644 --- a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java +++ b/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java @@ -1,4 +1,5 @@ import com.google.protobuf.ByteString; +import meerkat.bulletinboard.AsyncBulletinBoardClient; import meerkat.bulletinboard.BulletinBoardClient; import meerkat.bulletinboard.BulletinBoardClient.ClientCallback; import meerkat.bulletinboard.ThreadedBulletinBoardClient; @@ -97,7 +98,7 @@ public class BulletinBoardClientIntegrationTest { } } - private BulletinBoardClient bulletinBoardClient; + private AsyncBulletinBoardClient bulletinBoardClient; private PostCallback postCallback; private RedundancyCallback redundancyCallback; diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java index 945ae47..8131d2b 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java @@ -98,10 +98,9 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi + " REFERENCES MsgTable(EntryNum), FOREIGN KEY (TagId) REFERENCES TagTable(TagId), UNIQUE (EntryNum, TagID))"); list.add("CREATE TABLE IF NOT EXISTS SignatureTable (EntryNum INTEGER, SignerId BLOB, Signature BLOB," - + " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))"); + + " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum), UNIQUE(SignerId, EntryNum))"); list.add("CREATE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId)"); - list.add("CREATE UNIQUE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId, EntryNum)"); return list; } diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java new file mode 100644 index 0000000..04b7a06 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java @@ -0,0 +1,62 @@ +package meerkat.bulletinboard; + +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 14-Dec-15. + */ +public interface AsyncBulletinBoardClient extends BulletinBoardClient { + + /** + * Post a message to the bulletin board in an asynchronous manner + * @param msg is the message to be posted + * @param callback is a class containing methods to handle the result of the operation + * @return a unique message ID for the message, that can be later used to retrieve the batch + */ + MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback); + + /** + * This method allows for sending large messages as a batch to the bulletin board + * @param signerId is the canonical form for the ID of the sender of this batch + * @param batchId is a unique (per signer) ID for this batch + * @param batchDataList is the (canonically ordered) list of data comprising the batch message + * @param startPosition is the location (in the batch) of the first entry in batchDataList (optionally used to continue interrupted post operations) + * @param callback is a callback function class for handling results of the operation + * @return a unique message ID for the entire message, that can be later used to retrieve the batch + */ + MessageID postBatch(byte[] signerId, int batchId, List batchDataList, int startPosition, ClientCallback callback); + + /** + * Overloading of the postBatch method in which startPosition is set to the default value 0 + */ + MessageID postBatch(byte[] signerId, int batchId, List batchDataList, ClientCallback callback); + + /** + * Check how "safe" a given message is in an asynchronous manner + * The result of the computation is a rank between 0.0 and 1.0 indicating the fraction of servers containing the message + * @param id is the unique message identifier for retrieval + * @param callback is a callback function class for handling results of the operation + */ + void getRedundancy(MessageID id, ClientCallback callback); + + /** + * Read all messages posted matching the given filter in an asynchronous manner + * Note that if messages haven't been "fully posted", this might return a different + * set of messages in different calls. However, messages that are fully posted + * are guaranteed to be included. + * @param filterList return only messages that match the filters (null means no filtering). + * @param callback is a callback function class for handling results of the operation + */ + void readMessages(MessageFilterList filterList, ClientCallback> callback); + + /** + * Read a given batch message from the bulletin board + * @param signerId is the ID of the signer (sender) of the batch message + * @param batchId is the unique (per signer) ID of the batch + * @param callback is a callback class for handling the result of the operation + */ + void readBatch(byte[] signerId, int batchId, ClientCallback callback); + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java index c51e561..dcf6b15 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard; -import meerkat.comm.*; +import meerkat.comm.CommunicationException; import meerkat.protobuf.Voting.*; import static meerkat.protobuf.BulletinBoardAPI.*; @@ -24,26 +24,31 @@ public interface BulletinBoardClient { void init(BulletinBoardClientParams clientParams); /** - * Post a message to the bulletin board - * @param msg + * Post a message to the bulletin board in a synchronous manner + * @param msg is the message to be posted + * @return a unique message ID for the message, that can be later used to retrieve the batch + * @throws CommunicationException */ - MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback); + MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException; + + /** - * Check how "safe" a given message is - * @param id + * Check how "safe" a given message is in a synchronous manner + * @param id is the unique message identifier for retrieval * @return a normalized "redundancy score" from 0 (local only) to 1 (fully published) */ - void getRedundancy(MessageID id, ClientCallback callback); + float getRedundancy(MessageID id); /** - * Read all messages posted matching the given filter + * Read all messages posted matching the given filter in a synchronous manner * Note that if messages haven't been "fully posted", this might return a different * set of messages in different calls. However, messages that are fully posted * are guaranteed to be included. * @param filterList return only messages that match the filters (null means no filtering). + * @return the list of messages */ - void readMessages(MessageFilterList filterList, ClientCallback> callback); + List readMessages(MessageFilterList filterList); /** * Closes all connections, if any. diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/SignedBatch.java b/meerkat-common/src/main/java/meerkat/bulletinboard/SignedBatch.java new file mode 100644 index 0000000..46cf07e --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/SignedBatch.java @@ -0,0 +1,50 @@ +package meerkat.bulletinboard; + +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Crypto.*; + +import java.util.LinkedList; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 14-Dec-15. + * + * A data structure for holding both a batch message and its signature + */ +public class SignedBatch { + + private List batchDataList; + private Signature signature; + + public SignedBatch() { + batchDataList = new LinkedList(); + } + + public SignedBatch(List newDataList) { + this(); + appendBatchData(newDataList); + } + + public SignedBatch(List newDataList, Signature newSignature) { + this(newDataList); + signature = newSignature; + } + + public List getBatchDataList() { + return batchDataList; + } + + public Signature getSignature() { + return signature; + } + + public void appendBatchData(BatchData newBatchData) { + batchDataList.add(newBatchData); + } + + public void appendBatchData(List newBatchDataList) { + batchDataList.addAll(newBatchDataList); + } + + +} diff --git a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto index 0fe35f8..1a0bab1 100644 --- a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto +++ b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto @@ -77,4 +77,31 @@ message MessageFilterList { // To be implemented using intersection ("AND") operations. repeated MessageFilter filter = 1; +} + +// This message is used to start a batch transfer to the Bulletin Board Server +message BeginBatchMessage { + bytes signerId = 1; // Unique signer identifier + int32 batchId = 2; // Unique identifier for the batch (unique per signer) + repeated string tag = 3; // Tags for the batch message +} + +// This message is used to finalize and sign a batch transfer to the Bulletin Board Server +message CloseBatchMessage { + int32 batchId = 1; // Unique identifier for the batch (unique per signer) + int32 batchLength = 2; // Number of messages in the batch + meerkat.Signature sig = 3; // Signature on the (ordered) batch messages +} + +// Container for single batch message data +message BatchData { + bytes data = 1; +} + +// These messages comprise a batch message +message BatchMessage { + bytes signerId = 1; // Unique signer identifier + int32 batchId = 2; // Unique identifier for the batch (unique per signer) + int32 serialNum = 3; // Location of the message in the batch: starting from 0 + BatchData data = 4; // Actual data } \ No newline at end of file From 37fdc0bb83d9e8a14a54c960cb915d513dfd4766 Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Fri, 18 Dec 2015 14:39:40 +0200 Subject: [PATCH 02/12] Fixed minor H2 bug. Fixed dbTest gradle task (now tests all 3 supported DB engines). --- .../BulletinBoardClientIntegrationTest.java | 20 ++++++++--------- bulletin-board-server/build.gradle | 6 +++-- .../sqlserver/H2QueryProvider.java | 2 +- .../sqlserver/SQLiteQueryProvider.java | 22 ++++++++++++++++--- .../src/main/webapp/WEB-INF/web.xml | 2 +- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java b/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java index c090c92..55f0343 100644 --- a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java +++ b/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java @@ -27,6 +27,12 @@ public class BulletinBoardClientIntegrationTest { Semaphore jobSemaphore; Vector thrown; + protected void genericHandleFailure(Throwable t){ + System.err.println(t.getCause() + " " + t.getMessage()); + thrown.add(t); + jobSemaphore.release(); + } + private class PostCallback implements ClientCallback{ @Override @@ -37,8 +43,7 @@ public class BulletinBoardClientIntegrationTest { @Override public void handleFailure(Throwable t) { - thrown.add(t); - jobSemaphore.release(); + genericHandleFailure(t); } } @@ -59,8 +64,7 @@ public class BulletinBoardClientIntegrationTest { @Override public void handleFailure(Throwable t) { - thrown.add(t); - jobSemaphore.release(); + genericHandleFailure(t); } } @@ -93,8 +97,7 @@ public class BulletinBoardClientIntegrationTest { @Override public void handleFailure(Throwable t) { - thrown.add(t); - jobSemaphore.release(); + genericHandleFailure(t); } } @@ -202,10 +205,7 @@ public class BulletinBoardClientIntegrationTest { } bulletinBoardClient.close(); - - for (Throwable t : thrown) { - System.err.println(t.getMessage()); - } + if (thrown.size() > 0) { assert false; } diff --git a/bulletin-board-server/build.gradle b/bulletin-board-server/build.gradle index eb197c9..62e4b0c 100644 --- a/bulletin-board-server/build.gradle +++ b/bulletin-board-server/build.gradle @@ -75,13 +75,15 @@ dependencies { test { exclude '**/*SQLite*Test*' exclude '**/*H2*Test*' - exclude '**/*MySql*Test' + exclude '**/*MySQL*Test*' exclude '**/*IntegrationTest*' } task dbTest(type: Test) { include '**/*H2*Test*' - include '**/*MySql*Test' + include '**/*MySQL*Test*' + include '**/*SQLite*Test*' + outputs.upToDateWhen { false } } task integrationTest(type: Test) { diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java index fa2b146..d76601f 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java @@ -74,7 +74,7 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider case MAX_MESSAGES: return "LIMIT :Limit" + serialString; case MSG_ID: - return "MsgTable.MsgId = MsgId" + serialString; + return "MsgTable.MsgId = :MsgId" + serialString; case SIGNER_ID: return "EXISTS (SELECT 1 FROM SignatureTable" + " WHERE SignatureTable.SignerId = :SignerId" + serialString + " AND SignatureTable.EntryNum = MsgTable.EntryNum)"; diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java index 8131d2b..d796789 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java @@ -73,15 +73,31 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi @Override public String getConditionParamTypeName(FilterType filterType) throws IllegalArgumentException { - return null; //TODO: write this. + + switch(filterType) { + case EXACT_ENTRY: // Go through + case MAX_ENTRY: // Go through + case MAX_MESSAGES: + return "INTEGER"; + + case MSG_ID: // Go through + case SIGNER_ID: + return "BLOB"; + + case TAG: + return "VARCHAR"; + + default: + throw new IllegalArgumentException("Cannot serve a filter of type " + filterType); + } + } @Override public DataSource getDataSource() { - // TODO: Fix this + SQLiteDataSource dataSource = new SQLiteDataSource(); dataSource.setUrl("jdbc:sqlite:" + dbName); - dataSource.setDatabaseName("meerkat"); //TODO: Make generic return dataSource; } diff --git a/bulletin-board-server/src/main/webapp/WEB-INF/web.xml b/bulletin-board-server/src/main/webapp/WEB-INF/web.xml index 2198c07..226aa3b 100644 --- a/bulletin-board-server/src/main/webapp/WEB-INF/web.xml +++ b/bulletin-board-server/src/main/webapp/WEB-INF/web.xml @@ -31,7 +31,7 @@ mypass dbType - SQLite + H2 meerkat.bulletinboard.webapp.BulletinBoardWebApp From 37f962d520d9555e6336f8d27a1b34c89c71b019 Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Sat, 19 Dec 2015 19:54:50 +0200 Subject: [PATCH 03/12] Defined semi-final versions of the batch interfaces. Implemented in part extended BB Server interface. Added Digest support for Batch messages. Made GlobalCryptoSetup a final singleton. --- .../ThreadedBulletinBoardClient.java | 5 + .../callbacks/ClientFutureCallback.java | 6 - .../GetRedundancyFutureCallback.java | 6 +- .../callbacks/PostMessageFutureCallback.java | 8 +- .../callbacks/ReadMessagesFutureCallback.java | 9 +- .../BulletinBoardClientIntegrationTest.java | 3 +- .../sqlserver/BulletinBoardSQLServer.java | 203 +++++++++++++++--- .../sqlserver/MySQLQueryProvider.java | 80 ++++++- .../sqlserver/mappers/BatchDataMapper.java | 21 ++ .../{EntryNumMapper.java => LongMapper.java} | 2 +- .../webapp/BulletinBoardWebApp.java | 59 ++++- .../AsyncBulletinBoardClient.java | 28 ++- .../bulletinboard/BulletinBoardClient.java | 5 - .../bulletinboard/BulletinBoardServer.java | 73 +++++-- .../main/java/meerkat/crypto/BatchDigest.java | 20 ++ .../src/main/java/meerkat/crypto/Digest.java | 6 +- .../crypto/concrete/ECDSASignature.java | 5 +- .../crypto/concrete/ECElGamalEncryption.java | 5 +- .../crypto/concrete/GenericBatchDigest.java | 27 +++ .../crypto/concrete/GlobalCryptoSetup.java | 28 ++- .../meerkat/crypto/concrete/SHA256Digest.java | 3 +- .../main/proto/meerkat/BulletinBoardAPI.proto | 7 + .../crypto/concrete/ECElGamalUtils.java | 5 +- .../src/main/java/meerkat/rest/Constants.java | 4 + 24 files changed, 516 insertions(+), 102 deletions(-) create mode 100644 bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java rename bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/{EntryNumMapper.java => LongMapper.java} (87%) create mode 100644 meerkat-common/src/main/java/meerkat/crypto/BatchDigest.java create mode 100644 meerkat-common/src/main/java/meerkat/crypto/concrete/GenericBatchDigest.java diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java index dd103c6..5953bde 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java @@ -117,6 +117,11 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple // TODO: Implement } + @Override + public void subscribe(MessageFilterList filterList, MessageHandler messageHandler) { + // TODO: Implement + } + @Override public void close() { super.close(); diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java index 7c5b7b0..54cc63a 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java @@ -1,14 +1,8 @@ package meerkat.bulletinboard.callbacks; import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.BulletinClientJob; import meerkat.bulletinboard.BulletinClientJobResult; -import meerkat.bulletinboard.BulletinClientWorker; -import meerkat.protobuf.BulletinBoardAPI; - -import java.util.List; /** * This is a future callback used to listen to workers and run on job finish diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java index 518ed77..719428f 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java @@ -1,7 +1,7 @@ package meerkat.bulletinboard.callbacks; import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.BulletinBoardClient; +import meerkat.bulletinboard.AsyncBulletinBoardClient.*; import meerkat.bulletinboard.BulletinClientJobResult; import meerkat.protobuf.BulletinBoardAPI.*; @@ -13,10 +13,10 @@ import java.util.List; */ public class GetRedundancyFutureCallback extends ClientFutureCallback { - private BulletinBoardClient.ClientCallback callback; + private ClientCallback callback; public GetRedundancyFutureCallback(ListeningExecutorService listeningExecutor, - BulletinBoardClient.ClientCallback callback) { + ClientCallback callback) { super(listeningExecutor); this.callback = callback; } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java index 221ae1a..abd4247 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java @@ -2,13 +2,11 @@ package meerkat.bulletinboard.callbacks; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.BulletinBoardClient; +import meerkat.bulletinboard.AsyncBulletinBoardClient.*; import meerkat.bulletinboard.BulletinClientJob; import meerkat.bulletinboard.BulletinClientJobResult; import meerkat.bulletinboard.BulletinClientWorker; -import meerkat.protobuf.BulletinBoardAPI; -import java.util.List; /** * This is a future callback used to listen to workers and run on job finish @@ -16,10 +14,10 @@ import java.util.List; */ public class PostMessageFutureCallback extends ClientFutureCallback { - private BulletinBoardClient.ClientCallback callback; + private ClientCallback callback; public PostMessageFutureCallback(ListeningExecutorService listeningExecutor, - BulletinBoardClient.ClientCallback callback) { + ClientCallback callback) { super(listeningExecutor); this.callback = callback; } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java index 4c43ba2..808b7a6 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java @@ -1,11 +1,8 @@ package meerkat.bulletinboard.callbacks; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.BulletinBoardClient; -import meerkat.bulletinboard.BulletinClientJob; +import meerkat.bulletinboard.AsyncBulletinBoardClient.*; import meerkat.bulletinboard.BulletinClientJobResult; -import meerkat.bulletinboard.BulletinClientWorker; import meerkat.protobuf.BulletinBoardAPI; import java.util.List; @@ -16,10 +13,10 @@ import java.util.List; */ public class ReadMessagesFutureCallback extends ClientFutureCallback { - private BulletinBoardClient.ClientCallback> callback; + private ClientCallback> callback; public ReadMessagesFutureCallback(ListeningExecutorService listeningExecutor, - BulletinBoardClient.ClientCallback> callback) { + ClientCallback> callback) { super(listeningExecutor); this.callback = callback; } diff --git a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java b/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java index 55f0343..d7ae69c 100644 --- a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java +++ b/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java @@ -1,7 +1,6 @@ import com.google.protobuf.ByteString; import meerkat.bulletinboard.AsyncBulletinBoardClient; -import meerkat.bulletinboard.BulletinBoardClient; -import meerkat.bulletinboard.BulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; import meerkat.bulletinboard.ThreadedBulletinBoardClient; import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.protobuf.Crypto; diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java index 52bf42b..5f9b461 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java @@ -1,20 +1,29 @@ package meerkat.bulletinboard.sqlserver; +import java.security.InvalidKeyException; +import java.security.cert.CertificateException; import java.sql.*; import java.util.*; +import com.google.protobuf.ByteString; import com.google.protobuf.ProtocolStringList; import meerkat.bulletinboard.BulletinBoardServer; -import meerkat.bulletinboard.sqlserver.mappers.EntryNumMapper; -import meerkat.bulletinboard.sqlserver.mappers.MessageMapper; -import meerkat.bulletinboard.sqlserver.mappers.SignatureMapper; +import meerkat.bulletinboard.sqlserver.mappers.*; + import meerkat.comm.CommunicationException; + +import meerkat.crypto.BatchDigest; +import meerkat.crypto.DigitalSignature; +import meerkat.crypto.concrete.ECDSASignature; +import meerkat.crypto.concrete.SHA256Digest; + import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.protobuf.Crypto.Signature; import meerkat.protobuf.Crypto.SignatureVerificationKey; -import meerkat.crypto.Digest; -import meerkat.crypto.concrete.SHA256Digest; + + +import static meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider.*; import javax.sql.DataSource; @@ -23,6 +32,8 @@ import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.jdbc.support.GeneratedKeyHolder; import org.springframework.jdbc.support.KeyHolder; + + /** * This is a generic SQL implementation of the BulletinBoardServer API. */ @@ -46,7 +57,12 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ CONNECT_TAG(new String[] {"EntryNum","Tag"}), ADD_SIGNATURE(new String[] {"EntryNum","SignerId","Signature"}), GET_SIGNATURES(new String[] {"EntryNum"}), - GET_MESSAGES(new String[] {}); + GET_MESSAGES(new String[] {}), + GET_BATCH_MESSAGE_ENTRY(new String[] {"SignerId", "BatchId"}), + CHECK_BATCH_LENGTH(new String[] {"SignerId", "BatchId"}), + GET_BATCH_MESSAGE_DATA(new String[] {"SignerId", "BatchId", "StartPosition"}), + INSERT_BATCH_DATA(new String[] {"SignerId", "BatchId", "SerialNum", "Data"}), + CONNECT_BATCH_TAG(new String[] {"SignerId", "BatchId", "Tag"}); private String[] paramNames; @@ -58,6 +74,10 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ return paramNames; } + public String getParamName(int num) { + return paramNames[num]; + } + } /** @@ -152,6 +172,11 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } + /** + * This method returns the value of the parameter specified in a message filter + * @param messageFilter is the filter + * @return the object parameter for the SQL query embedded in the filter (this depends on the filter type) + */ private Object getParam(MessageFilter messageFilter) { switch (messageFilter.getType()) { @@ -170,7 +195,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ case MAX_MESSAGES: return messageFilter.getMaxMessages(); - default: + default: // Unsupported filter type return null; } @@ -193,7 +218,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ protected NamedParameterJdbcTemplate jdbcTemplate; - protected Digest digest; + protected BatchDigest digest; + protected DigitalSignature signer; protected List trusteeSignatureVerificationArray; protected int minTrusteeSignatures; @@ -232,6 +258,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ // TODO write signature reading part. digest = new SHA256Digest(); + signer = new ECDSASignature(); jdbcTemplate = new NamedParameterJdbcTemplate(sqlQueryProvider.getDataSource()); @@ -264,12 +291,12 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ String sql; - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.INSERT_NEW_TAG); + sql = sqlQueryProvider.getSQLString(QueryType.INSERT_NEW_TAG); Map namedParameters[] = new HashMap[tags.length]; for (int i = 0 ; i < tags.length ; i++){ namedParameters[i] = new HashMap(); - namedParameters[i].put("Tag", tags[i]); + namedParameters[i].put(QueryType.INSERT_NEW_TAG.getParamName(0), tags[i]); } jdbcTemplate.batchUpdate(sql, namedParameters); @@ -316,11 +343,11 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ // Add message to table if needed and store entry number of message. - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.FIND_MSG_ID); + sql = sqlQueryProvider.getSQLString(QueryType.FIND_MSG_ID); Map namedParameters = new HashMap(); - namedParameters.put("MsgId",msgID); + namedParameters.put(QueryType.FIND_MSG_ID.getParamName(0),msgID); - List entryNums = jdbcTemplate.query(sql, new MapSqlParameterSource(namedParameters), new EntryNumMapper()); + List entryNums = jdbcTemplate.query(sql, new MapSqlParameterSource(namedParameters), new LongMapper()); if (entryNums.size() > 0){ @@ -328,8 +355,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } else{ - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.INSERT_MSG); - namedParameters.put("Msg", msg.getMsg().toByteArray()); + sql = sqlQueryProvider.getSQLString(QueryType.INSERT_MSG); + namedParameters.put(QueryType.INSERT_MSG.getParamName(0), msg.getMsg().toByteArray()); KeyHolder keyHolder = new GeneratedKeyHolder(); jdbcTemplate.update(sql,new MapSqlParameterSource(namedParameters),keyHolder); @@ -354,14 +381,14 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ // Connect message to tags. - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.CONNECT_TAG); + sql = sqlQueryProvider.getSQLString(QueryType.CONNECT_TAG); namedParameterArray = new HashMap[tags.length]; for (int i = 0 ; i < tags.length ; i++) { namedParameterArray[i] = new HashMap(); - namedParameterArray[i].put("EntryNum", entryNum); - namedParameterArray[i].put("Tag", tags[i]); + namedParameterArray[i].put(QueryType.CONNECT_TAG.getParamName(0), entryNum); + namedParameterArray[i].put(QueryType.CONNECT_TAG.getParamName(1), tags[i]); } jdbcTemplate.batchUpdate(sql, namedParameterArray); @@ -374,15 +401,15 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ // Connect message to signatures. - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.ADD_SIGNATURE); + sql = sqlQueryProvider.getSQLString(QueryType.ADD_SIGNATURE); namedParameterArray = new HashMap[signatures.length]; for (int i = 0 ; i < signatures.length ; i++) { namedParameterArray[i] = new HashMap(); - namedParameterArray[i].put("EntryNum", entryNum); - namedParameterArray[i].put("SignerId", signatures[i].getSignerId().toByteArray()); - namedParameterArray[i].put("Signature", signatures[i].toByteArray()); + namedParameterArray[i].put(QueryType.ADD_SIGNATURE.getParamName(0), entryNum); + namedParameterArray[i].put(QueryType.ADD_SIGNATURE.getParamName(1), signatures[i].getSignerId().toByteArray()); + namedParameterArray[i].put(QueryType.ADD_SIGNATURE.getParamName(2), signatures[i].toByteArray()); } jdbcTemplate.batchUpdate(sql,namedParameterArray); @@ -412,7 +439,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ // Check if Tag/Signature tables are required for filtering purposes - sqlBuilder.append(sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.GET_MESSAGES)); + sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.GET_MESSAGES)); // Add conditions namedParameters = new MapSqlParameterSource(); @@ -434,7 +461,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ sqlBuilder.append(sqlQueryProvider.getCondition(filter.getType(), paramNum)); - SQLQueryProvider.FilterTypeParam filterTypeParam = SQLQueryProvider.FilterTypeParam.getFilterTypeParamName(filter.getType()); + FilterTypeParam filterTypeParam = FilterTypeParam.getFilterTypeParamName(filter.getType()); namedParameters.addValue( filterTypeParam.getParamName() + Integer.toString(paramNum), @@ -457,10 +484,10 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ // Retrieve signatures namedParameters = new MapSqlParameterSource(); - namedParameters.addValue("EntryNum", msgBuilder.getEntryNum()); + namedParameters.addValue(QueryType.GET_SIGNATURES.getParamName(0), msgBuilder.getEntryNum()); List signatures = jdbcTemplate.query( - sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.GET_SIGNATURES), + sqlQueryProvider.getSQLString(QueryType.GET_SIGNATURES), namedParameters, signatureMapper); @@ -478,6 +505,130 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } + /** + * This method checks if a specified batch exists and is already closed + * @param signerId is the ID of the publisher of the batch + * @param batchId is the unique (per signer) batch ID + * @return TRUE if the batch is closed and FALSE if it is still open or doesn't exist at all + */ + private boolean isBatchClosed(ByteString signerId, int batchId){ + + String sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_ENTRY); + + MapSqlParameterSource namedParameters = new MapSqlParameterSource(); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0), signerId); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1), batchId); + + List result = jdbcTemplate.query(sql,namedParameters,new LongMapper()); + + return (result.size() > 0); + + } + + @Override + public BoolMsg beginBatch(BeginBatchMessage message) throws CommunicationException { + + // Check if batch is closed + if (isBatchClosed(message.getSignerId(), message.getBatchId())) { + return BoolMsg.newBuilder().setValue(false).build(); + } + + // Add new tags to table + ProtocolStringList tagList = message.getTagList(); + String[] tags = new String[tagList.size()]; + tags = tagList.toArray(tags); + try { + insertNewTags(tags); + } catch (SQLException e) { + throw new CommunicationException(e.getMessage()); + } + + // Connect tags + String sql = sqlQueryProvider.getSQLString(QueryType.CONNECT_BATCH_TAG); + MapSqlParameterSource namedParameters[] = new MapSqlParameterSource[tags.length]; + + for (int i=0 ; i < tags.length ; i++) { + namedParameters[i] = new MapSqlParameterSource(); + namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(0),message.getSignerId()); + namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(1),message.getBatchId()); + namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(2),tags[i]); + } + + jdbcTemplate.batchUpdate(sql,namedParameters); + + return BoolMsg.newBuilder().setValue(true).build(); + } + + @Override + public BoolMsg postBatchMessage(BatchMessage batchMessage) { + + // Check if batch is closed + if (isBatchClosed(batchMessage.getSignerId(), batchMessage.getBatchId())) { + return BoolMsg.newBuilder().setValue(false).build(); + } + + // Add data + String sql = sqlQueryProvider.getSQLString(QueryType.INSERT_BATCH_DATA); + MapSqlParameterSource namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(0),batchMessage.getSignerId()); + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(1),batchMessage.getBatchId()); + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(2),batchMessage.getSerialNum()); + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(3),batchMessage.getData()); + + jdbcTemplate.update(sql, namedParameters); + + return BoolMsg.newBuilder().setValue(true).build(); + + } + + @Override + public BoolMsg closeBatchMessage(CloseBatchMessage message) throws CommunicationException { + + // Check batch size + + String sql = sqlQueryProvider.getSQLString(QueryType.CHECK_BATCH_LENGTH); + MapSqlParameterSource namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(0),message.getSig().getSignerId()); + namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(1),message.getBatchId()); + + List lengthResult = jdbcTemplate.query(sql, namedParameters, new LongMapper()); + + if (lengthResult.get(0) != message.getBatchLength()) { + return BoolMsg.newBuilder().setValue(false).build(); + } + + // Check signature + + sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA); + namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),message.getSig().getSignerId()); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),message.getBatchId()); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),0); // Read from the beginning + + List batchDataList = jdbcTemplate.query(sql, namedParameters, new BatchDataMapper()); + + try { + signer.initVerify(message.getSig()); + } catch (CertificateException | InvalidKeyException e) { + return BoolMsg.newBuilder().setValue(false).build(); + } + + //TODO: FInish this + + //signer.updateContent(msgStream); + //assertTrue("Signature did not verify!", signer.verify()); + + return null; + } + + @Override + public List readBatch(BatchSpecificationMessage message) { + return null; // TODO: Implement this + } + @Override public void close() {} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java index c00c044..0093bd5 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java @@ -5,6 +5,7 @@ import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider; import meerkat.protobuf.BulletinBoardAPI.FilterType; import javax.sql.DataSource; +import java.text.MessageFormat; import java.util.LinkedList; import java.util.List; @@ -32,21 +33,72 @@ public class MySQLQueryProvider implements SQLQueryProvider { public String getSQLString(QueryType queryType) throws IllegalArgumentException{ switch(queryType) { + case ADD_SIGNATURE: - return "INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:EntryNum, :SignerId, :Signature)"; + return MessageFormat.format( + "INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES ({0}, {1}, {2})", + QueryType.ADD_SIGNATURE.getParamName(0), + QueryType.ADD_SIGNATURE.getParamName(1), + QueryType.ADD_SIGNATURE.getParamName(2)); + case CONNECT_TAG: - return "INSERT IGNORE INTO MsgTagTable (TagId, EntryNum)" - + " SELECT TagTable.TagId, :EntryNum AS EntryNum FROM TagTable WHERE Tag = :Tag"; + return MessageFormat.format( + "INSERT IGNORE INTO MsgTagTable (TagId, EntryNum)" + + " SELECT TagTable.TagId, {0} AS EntryNum FROM TagTable WHERE Tag = {1}", + QueryType.CONNECT_TAG.getParamName(0), + QueryType.CONNECT_TAG.getParamName(1)); + case FIND_MSG_ID: - return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId"; + return MessageFormat.format( + "SELECT EntryNum From MsgTable WHERE MsgId = {0}", + QueryType.FIND_MSG_ID.getParamName(0)); + case GET_MESSAGES: return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"; + case GET_SIGNATURES: - return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum"; + return MessageFormat.format( + "SELECT Signature FROM SignatureTable WHERE EntryNum = {0}", + QueryType.GET_SIGNATURES.getParamName(0)); + case INSERT_MSG: - return "INSERT INTO MsgTable (MsgId, Msg) VALUES(:MsgId, :Msg)"; + return MessageFormat.format( + "INSERT INTO MsgTable (MsgId, Msg) VALUES({0}, {1})", + QueryType.INSERT_MSG.getParamName(0), + QueryType.INSERT_MSG.getParamName(1)); + case INSERT_NEW_TAG: - return "INSERT IGNORE INTO TagTable(Tag) VALUES (:Tag)"; + return MessageFormat.format( + "INSERT IGNORE INTO TagTable(Tag) VALUES ({0})", + QueryType.INSERT_NEW_TAG.getParamName(0)); + + case GET_BATCH_MESSAGE_ENTRY: + return MessageFormat.format( + "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable" + + "INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum" + + "INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum" + + "INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId" + + "WHERE SignatureTable.SignerId = {0}" + + "AND TagTable.Tag = {1}", + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0), + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1)); + + case GET_BATCH_MESSAGE_DATA: + return MessageFormat.format( + "SELECT Data FROM BatchTable" + + " WHERE SignerId = {0} AND BatchId = {1} AND SerialNum >= {2}" + + " ORDER BY SerialNum ASC", + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2)); + + case CHECK_BATCH_LENGTH: + return MessageFormat.format( + "SELECT COUNT(Data) AS BatchLength FROM BatchTable" + + " WHERE SignerId = {0} AND BatchId = {1}", + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1)); + default: throw new IllegalArgumentException("Cannot serve a query of type " + queryType); } @@ -119,7 +171,8 @@ public class MySQLQueryProvider implements SQLQueryProvider { public List getSchemaCreationCommands() { List list = new LinkedList(); - list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB, Msg BLOB, UNIQUE(MsgId(50)))"); + list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY," + + " MsgId TINYBLOB, Msg BLOB, UNIQUE(MsgId(50)))"); list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INT NOT NULL AUTO_INCREMENT PRIMARY KEY, Tag VARCHAR(50), UNIQUE(Tag))"); @@ -129,7 +182,14 @@ public class MySQLQueryProvider implements SQLQueryProvider { + " CONSTRAINT UNIQUE (EntryNum, TagID))"); list.add("CREATE TABLE IF NOT EXISTS SignatureTable (EntryNum INT, SignerId TINYBLOB, Signature TINYBLOB," - + " INDEX(SignerId(32)), CONSTRAINT Uni UNIQUE(SignerId(32), EntryNum), CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))"); + + " INDEX(SignerId(32)), CONSTRAINT Unique_Signature UNIQUE(SignerId(32), EntryNum)," + + " CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))"); + + list.add("CREATE TABLE IF NOT EXISTS BatchTable (SignerId TINYBLOB, BatchId INT, SerialNum INT, Data BLOB," + + " CONSTRAINT Unique_Batch UNIQUE(SignerId(32), BatchId, SerialNum))"); + + list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (SignerId TINYBLOB, BatchId INT, TagId INT," + + " INDEX(SignerId, BatchId), CONSTRAINT FOREIGN KEY (TagId) REFERENCES TagTable(TagId))"); return list; } @@ -138,6 +198,8 @@ public class MySQLQueryProvider implements SQLQueryProvider { public List getSchemaDeletionCommands() { List list = new LinkedList(); + list.add("DROP TABLE IF EXISTS BatchTagTable"); + list.add("DROP TABLE IF EXISTS BatchTable"); list.add("DROP TABLE IF EXISTS MsgTagTable"); list.add("DROP TABLE IF EXISTS SignatureTable"); list.add("DROP TABLE IF EXISTS TagTable"); diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java new file mode 100644 index 0000000..0189584 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java @@ -0,0 +1,21 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import com.google.protobuf.ByteString; +import meerkat.protobuf.BulletinBoardAPI.BatchData; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by Arbel Deutsch Peled on 19-Dec-15. + */ +public class BatchDataMapper implements RowMapper { + + @Override + public BatchData mapRow(ResultSet rs, int rowNum) throws SQLException { + + return BatchData.newBuilder().setData(ByteString.copyFrom(rs.getBytes(rowNum))).build(); + + } +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/EntryNumMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/LongMapper.java similarity index 87% rename from bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/EntryNumMapper.java rename to bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/LongMapper.java index 478c39e..1ec0d98 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/EntryNumMapper.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/LongMapper.java @@ -9,7 +9,7 @@ import java.sql.SQLException; /** * Created by Arbel Deutsch Peled on 11-Dec-15. */ -public class EntryNumMapper implements RowMapper { +public class LongMapper implements RowMapper { @Override public Long mapRow(ResultSet rs, int rowNum) throws SQLException { diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java index b3fc03c..27d7bd5 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java @@ -17,12 +17,11 @@ import meerkat.bulletinboard.sqlserver.H2QueryProvider; import meerkat.bulletinboard.sqlserver.MySQLQueryProvider; import meerkat.bulletinboard.sqlserver.SQLiteQueryProvider; import meerkat.comm.CommunicationException; -import meerkat.protobuf.BulletinBoardAPI.BoolMsg; -import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; -import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessageList; -import meerkat.protobuf.BulletinBoardAPI.MessageFilterList; +import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.rest.Constants; +import java.util.List; + @Path(Constants.BULLETIN_BOARD_SERVER_PATH) public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextListener{ @@ -99,6 +98,58 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL return bulletinBoard.readMessages(filterList); } + @Path(Constants.BEGIN_BATCH_PATH) + @POST + @Consumes(Constants.MEDIATYPE_PROTOBUF) + @Produces(Constants.MEDIATYPE_PROTOBUF) + @Override + public BoolMsg beginBatch(BeginBatchMessage message) { + try { + return bulletinBoard.beginBatch(message); + } catch (CommunicationException e) { + System.err.println(e.getMessage()); + return null; + } + } + + @Path(Constants.POST_BATCH_PATH) + @POST + @Consumes(Constants.MEDIATYPE_PROTOBUF) + @Produces(Constants.MEDIATYPE_PROTOBUF) + @Override + public BoolMsg postBatchMessage(BatchMessage batchMessage) { + try { + return bulletinBoard.postBatchMessage(batchMessage); + } catch (CommunicationException e) { + System.err.println(e.getMessage()); + return null; + } + } + + @Path(Constants.CLOSE_BATCH_PATH) + @POST + @Consumes(Constants.MEDIATYPE_PROTOBUF) + @Produces(Constants.MEDIATYPE_PROTOBUF) + @Override + public BoolMsg closeBatchMessage(CloseBatchMessage message) { + try { + return bulletinBoard.closeBatchMessage(message); + } catch (CommunicationException e) { + System.err.println(e.getMessage()); + return null; + } + } + + @Override + public List readBatch(BatchSpecificationMessage message) { + try { + return bulletinBoard.readBatch(message); + } catch (CommunicationException | IllegalArgumentException e) { + System.err.println(e.getMessage()); + return null; + } + } + @Override public void close(){ try { diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java index 04b7a06..4d8075e 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java @@ -9,13 +9,22 @@ import java.util.List; */ public interface AsyncBulletinBoardClient extends BulletinBoardClient { + public interface ClientCallback { + void handleCallback(T msg); + void handleFailure(Throwable t); + } + + public interface MessageHandler { + void handleNewMessages(List messageList); + } + /** * Post a message to the bulletin board in an asynchronous manner * @param msg is the message to be posted * @param callback is a class containing methods to handle the result of the operation * @return a unique message ID for the message, that can be later used to retrieve the batch */ - MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback); + public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback); /** * This method allows for sending large messages as a batch to the bulletin board @@ -26,12 +35,12 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param callback is a callback function class for handling results of the operation * @return a unique message ID for the entire message, that can be later used to retrieve the batch */ - MessageID postBatch(byte[] signerId, int batchId, List batchDataList, int startPosition, ClientCallback callback); + public MessageID postBatch(byte[] signerId, int batchId, List batchDataList, int startPosition, ClientCallback callback); /** * Overloading of the postBatch method in which startPosition is set to the default value 0 */ - MessageID postBatch(byte[] signerId, int batchId, List batchDataList, ClientCallback callback); + public MessageID postBatch(byte[] signerId, int batchId, List batchDataList, ClientCallback callback); /** * Check how "safe" a given message is in an asynchronous manner @@ -39,7 +48,7 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param id is the unique message identifier for retrieval * @param callback is a callback function class for handling results of the operation */ - void getRedundancy(MessageID id, ClientCallback callback); + public void getRedundancy(MessageID id, ClientCallback callback); /** * Read all messages posted matching the given filter in an asynchronous manner @@ -49,7 +58,7 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param filterList return only messages that match the filters (null means no filtering). * @param callback is a callback function class for handling results of the operation */ - void readMessages(MessageFilterList filterList, ClientCallback> callback); + public void readMessages(MessageFilterList filterList, ClientCallback> callback); /** * Read a given batch message from the bulletin board @@ -57,6 +66,13 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param batchId is the unique (per signer) ID of the batch * @param callback is a callback class for handling the result of the operation */ - void readBatch(byte[] signerId, int batchId, ClientCallback callback); + public void readBatch(byte[] signerId, int batchId, ClientCallback callback); + + /** + * Subscribes to a notifier that will return any new messages on the server that match the given filters + * @param filterList defines the set of filters for message retrieval + * @param messageHandler defines the handler for new messages received + */ + public void subscribe(MessageFilterList filterList, MessageHandler messageHandler); } diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java index dcf6b15..2f5a3df 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java @@ -12,11 +12,6 @@ import java.util.List; */ public interface BulletinBoardClient { - interface ClientCallback { - void handleCallback(T msg); - void handleFailure(Throwable t); - } - /** * Initialize the client to use some specified servers * @param clientParams contains the parameters required for the client setup diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java index da53c1f..70721a7 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java @@ -3,40 +3,83 @@ package meerkat.bulletinboard; import meerkat.comm.CommunicationException; import meerkat.protobuf.BulletinBoardAPI.*; +import java.util.List; + /** * Created by Arbel on 07/11/15. * - * This interface refers to a single instance of a Bulletin Board. - * An implementation of this interface may use any DB and be hosted on any machine. + * This interface refers to a single instance of a Bulletin Board + * An implementation of this interface may use any DB and be hosted on any machine. */ public interface BulletinBoardServer{ /** - * This method initializes the server by reading the signature data and storing it. - * It also establishes the connection to the DB. - * @throws CommunicationException on DB connection error. + * This method initializes the server by reading the signature data and storing it + * It also establishes the connection to the DB + * @throws CommunicationException on DB connection error */ public void init(String meerkatDB) throws CommunicationException; /** * Post a message to bulletin board. * @param msg is the actual (signed) message - * @return TRUE if the message has been authenticated and FALSE otherwise (in ProtoBuf form). - * @throws CommunicationException on DB connection error. + * @return TRUE if the message has been authenticated and FALSE otherwise (in ProtoBuf form) + * @throws CommunicationException on DB connection error */ - public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException; + public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException; /** - * Read all messages posted matching the given filter. - * @param filter return only messages that match the filter (empty list means no filtering). + * Read all messages posted matching the given filter + * @param filterList return only messages that match the filters (empty list or null means no filtering) * @return + * @throws CommunicationException on DB connection error */ - BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException; - + public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException; + + /** + * Informs server about a new batch message + * @param message contains the required data about the new batch + * @return TRUE if the batch request is accepted amd FALSE otherwise + * Specifically, if such a batch already exists and is not yet closed: the value returned will be TRUE + * However, if such a batch exists and is already closed: the value returned will be FALSE + * @throws CommunicationException on DB connection error + */ + public BoolMsg beginBatch(BeginBatchMessage message) throws CommunicationException; + + /** + * Posts a (part of a) batch message to the bulletin board + * Note that the existence and contents of a batch message are not available for reading before the batch is finalized + * @param batchMessage contains the (partial) data this message carries as well as meta-data required in order to place the data + * in the correct position inside the correct batch + * @return TRUE if the message is accepted and successfully saved and FALSE otherwise + * Specifically, if the batch is already closed: the value returned will be FALSE + * However, requiring to open a batch before insertion of messages is implementation-dependent + * @throws CommunicationException on DB connection error + */ + public BoolMsg postBatchMessage(BatchMessage batchMessage) throws CommunicationException; + + /** + * Attempts to close and finalize a batch message + * @param message contains the data necessary to close the batch; in particular: the signature for the batch + * @return TRUE if the batch was successfully closed, FALSE otherwise + * Specifically, if the signature is invalid or if some of the batch parts have not yet been submitted: the value returned will be FALSE + * @throws CommunicationException on DB connection error + */ + public BoolMsg closeBatchMessage(CloseBatchMessage message) throws CommunicationException; + + /** + * Reads a batch message from the server (starting with the supplied position) + * @param message specifies the signer ID and the batch ID to read as well as an (optional) start position + * @return an ordered list of batch messages starting from the specified start position (if given) or from the beginning (if omitted) + * @throws CommunicationException on DB connection error + * @throws IllegalArgumentException if message does not specify a batch + */ + public List readBatch(BatchSpecificationMessage message) throws CommunicationException, IllegalArgumentException; + /** - * This method closes the connection to the DB. - * @throws CommunicationException on DB connection error. + * This method closes the connection to the DB + * @throws CommunicationException on DB connection error */ - public void close() throws CommunicationException; + public void close() throws CommunicationException; } diff --git a/meerkat-common/src/main/java/meerkat/crypto/BatchDigest.java b/meerkat-common/src/main/java/meerkat/crypto/BatchDigest.java new file mode 100644 index 0000000..ff257b2 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/crypto/BatchDigest.java @@ -0,0 +1,20 @@ +package meerkat.crypto; + +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 18-Dec-15. + * Extends the Digest interface with a method for digesting Batch messages + */ +public interface BatchDigest extends Digest{ + + /** + * Update the digest with the + * @param beginBatchMessage is the message that starts the batch + * @param batchDataList is the (ordered) list of data in the batch + */ + public void update(BeginBatchMessage beginBatchMessage, List batchDataList); + +} diff --git a/meerkat-common/src/main/java/meerkat/crypto/Digest.java b/meerkat-common/src/main/java/meerkat/crypto/Digest.java index 06b012c..c72206e 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/Digest.java +++ b/meerkat-common/src/main/java/meerkat/crypto/Digest.java @@ -13,7 +13,7 @@ public interface Digest { * (copied from {@link MessageDigest#digest()}) * @return */ - byte[] digest(); + public byte[] digest(); /** * Updates the digest using the specified message (in serialized wire form) @@ -22,12 +22,12 @@ public interface Digest { * @param msg * @return */ - void update(Message msg); + public void update(Message msg); /** * Resets the digest for further use. */ - void reset(); + public void reset(); /** * Clone the current digest state diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java index 887b8e8..28e4600 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java @@ -30,7 +30,10 @@ import javax.security.auth.callback.UnsupportedCallbackException; * * This class is not thread-safe (each thread should have its own instance). */ -public class ECDSASignature extends GlobalCryptoSetup implements DigitalSignature { +public class ECDSASignature implements DigitalSignature { + + private static GlobalCryptoSetup globalCryptoSetup = GlobalCryptoSetup.getInstance(); + final Logger logger = LoggerFactory.getLogger(getClass()); final public static String KEYSTORE_TYPE = "PKCS12"; diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java index f9936c7..98d32bf 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java @@ -31,7 +31,10 @@ import java.util.Random; /** * Created by talm on 17/11/15. */ -public class ECElGamalEncryption extends GlobalCryptoSetup implements Encryption { +public class ECElGamalEncryption implements Encryption { + + private static GlobalCryptoSetup globalCryptoSetup = GlobalCryptoSetup.getInstance(); + final Logger logger = LoggerFactory.getLogger(getClass()); public final static String KEY_ALGORITHM = "ECDH"; diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/GenericBatchDigest.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/GenericBatchDigest.java new file mode 100644 index 0000000..38b2192 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/GenericBatchDigest.java @@ -0,0 +1,27 @@ +package meerkat.crypto.concrete; + +import com.google.protobuf.Message; +import meerkat.crypto.BatchDigest; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 19-Dec-15. + */ +public abstract class GenericBatchDigest implements BatchDigest{ + + + @Override + public void update(BeginBatchMessage beginBatchMessage, List batchDataList) { + update(beginBatchMessage); + for (BatchData batchData : batchDataList) { + update(batchData); + } + } + + @Override + // Repeated here to circumvent compiler error + public abstract BatchDigest clone() throws CloneNotSupportedException; + +} diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/GlobalCryptoSetup.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/GlobalCryptoSetup.java index 4f2e7a5..3d12701 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/GlobalCryptoSetup.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/GlobalCryptoSetup.java @@ -10,13 +10,26 @@ import java.security.Security; /** * A class that performs required crypto setup */ -public class GlobalCryptoSetup { +public final class GlobalCryptoSetup { + + private static GlobalCryptoSetup globalCryptoSetup; + final static Logger logger = LoggerFactory.getLogger(GlobalCryptoSetup.class); - static boolean loadedBouncyCastle = false; - static Provider bouncyCastleProvider; + private boolean loadedBouncyCastle = false; + private Provider bouncyCastleProvider; - public static boolean hasSecp256k1Curve() { + private GlobalCryptoSetup() { doSetup(); } + + public static GlobalCryptoSetup getInstance() { + if (globalCryptoSetup == null) { + globalCryptoSetup = new GlobalCryptoSetup(); + } + + return globalCryptoSetup; + } + + public boolean hasSecp256k1Curve() { // For now we just check if the java version is at least 8 String[] version = System.getProperty("java.version").split("\\."); int major = Integer.parseInt(version[0]); @@ -24,9 +37,11 @@ public class GlobalCryptoSetup { return ((major > 1) || ((major > 0) && (minor > 7))); } - public static Provider getBouncyCastleProvider() { doSetup(); return bouncyCastleProvider; } + public Provider getBouncyCastleProvider() { + return bouncyCastleProvider; + } - public static synchronized void doSetup() { + public void doSetup() { if (bouncyCastleProvider == null) { bouncyCastleProvider = new BouncyCastleProvider(); // Make bouncycastle our default provider if we're running on a JVM version < 8 @@ -39,5 +54,4 @@ public class GlobalCryptoSetup { } } - public GlobalCryptoSetup() { doSetup(); } } diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java index 4f60af3..ac58f3a 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java @@ -2,6 +2,7 @@ package meerkat.crypto.concrete; import com.google.protobuf.ByteString; import com.google.protobuf.Message; +import meerkat.crypto.BatchDigest; import meerkat.crypto.Digest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,7 +14,7 @@ import java.security.NoSuchAlgorithmException; /** * Created by talm on 11/9/15. */ -public class SHA256Digest extends GlobalCryptoSetup implements Digest { +public class SHA256Digest extends GenericBatchDigest { final Logger logger = LoggerFactory.getLogger(getClass()); public static final String SHA256 = "SHA-256"; diff --git a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto index 1a0bab1..2ae4068 100644 --- a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto +++ b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto @@ -104,4 +104,11 @@ message BatchMessage { int32 batchId = 2; // Unique identifier for the batch (unique per signer) int32 serialNum = 3; // Location of the message in the batch: starting from 0 BatchData data = 4; // Actual data +} + +// This message defines which batch to read and from which location to start reading +message BatchSpecificationMessage { + bytes signerId = 1; // Unique signer identifier + int32 batchId = 2; // Unique identifier for the batch (unique per signer) + int32 startPosition = 3; // Position in batch to start reading from } \ No newline at end of file diff --git a/meerkat-common/src/test/java/meerkat/crypto/concrete/ECElGamalUtils.java b/meerkat-common/src/test/java/meerkat/crypto/concrete/ECElGamalUtils.java index 66e1647..e57c817 100644 --- a/meerkat-common/src/test/java/meerkat/crypto/concrete/ECElGamalUtils.java +++ b/meerkat-common/src/test/java/meerkat/crypto/concrete/ECElGamalUtils.java @@ -26,6 +26,9 @@ import java.security.spec.InvalidKeySpecException; * utilities for ECElgamal */ public class ECElGamalUtils { + + private static GlobalCryptoSetup globalCryptoSetup = GlobalCryptoSetup.getInstance(); + final static Logger logger = LoggerFactory.getLogger(ECElGamalUtils.class); public final static String ENCRYPTION_KEY_ALGORITHM = "ECDH"; @@ -43,7 +46,7 @@ public class ECElGamalUtils { try { KeyFactory fact = KeyFactory.getInstance(ENCRYPTION_KEY_ALGORITHM, - GlobalCryptoSetup.getBouncyCastleProvider()); + globalCryptoSetup.getBouncyCastleProvider()); PublicKey javaPk = fact.generatePublic(pubKeySpec); ConcreteCrypto.ElGamalPublicKey serializedPk = ConcreteCrypto.ElGamalPublicKey.newBuilder() .setSubjectPublicKeyInfo(ByteString.copyFrom(javaPk.getEncoded())).build(); diff --git a/restful-api-common/src/main/java/meerkat/rest/Constants.java b/restful-api-common/src/main/java/meerkat/rest/Constants.java index 73ed7d1..8215641 100644 --- a/restful-api-common/src/main/java/meerkat/rest/Constants.java +++ b/restful-api-common/src/main/java/meerkat/rest/Constants.java @@ -9,4 +9,8 @@ public interface Constants { public static final String BULLETIN_BOARD_SERVER_PATH = "/bbserver"; public static final String READ_MESSAGES_PATH = "/readmessages"; public static final String POST_MESSAGE_PATH = "/postmessage"; + public static final String BEGIN_BATCH_PATH = "/beginbatch"; + public static final String POST_BATCH_PATH = "/postbatch"; + public static final String CLOSE_BATCH_PATH = "/closebatch"; + } From b5237d6c9f32b3946cb760bd2ffabe2ed1706dcb Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Mon, 21 Dec 2015 23:16:06 +0200 Subject: [PATCH 04/12] Implemented (untested) batch messages in Bulletin Board Server (MySQL implementation only). Implemented generic batch message signatures and digests. Created new interface for Bulletin Board constants. --- .../bulletinboard/BulletinClientWorker.java | 9 +- .../SimpleBulletinBoardClient.java | 8 +- .../ThreadedBulletinBoardClient.java | 2 +- .../sqlserver/BulletinBoardSQLServer.java | 117 ++++++++++++++---- .../sqlserver/MySQLQueryProvider.java | 46 +++++-- .../sqlserver/mappers/BatchDataMapper.java | 2 +- .../sqlserver/mappers/StringMapper.java | 18 +++ .../webapp/BulletinBoardWebApp.java | 35 +++--- ...BulletinBoardSQLServerIntegrationTest.java | 9 +- .../AsyncBulletinBoardClient.java | 2 +- .../meerkat/bulletinboard/BatchDigest.java | 20 +++ .../bulletinboard/BatchDigitalSignature.java | 34 +++++ .../bulletinboard/BulletinBoardConstants.java | 21 ++++ .../{SignedBatch.java => CompleteBatch.java} | 26 ++-- .../bulletinboard/GenericBatchDigest.java | 54 ++++++++ .../GenericBatchDigitalSignature.java | 95 ++++++++++++++ .../main/java/meerkat/crypto/BatchDigest.java | 20 --- .../crypto/concrete/GenericBatchDigest.java | 27 ---- .../meerkat/crypto/concrete/SHA256Digest.java | 3 +- .../src/main/java/meerkat/rest/Constants.java | 8 -- 20 files changed, 428 insertions(+), 128 deletions(-) create mode 100644 bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/StringMapper.java create mode 100644 meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigest.java create mode 100644 meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigitalSignature.java create mode 100644 meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java rename meerkat-common/src/main/java/meerkat/bulletinboard/{SignedBatch.java => CompleteBatch.java} (50%) create mode 100644 meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java create mode 100644 meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java delete mode 100644 meerkat-common/src/main/java/meerkat/crypto/BatchDigest.java delete mode 100644 meerkat-common/src/main/java/meerkat/crypto/concrete/GenericBatchDigest.java diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java index 51599d5..f03ab31 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java @@ -8,6 +8,7 @@ import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.rest.Constants; import meerkat.rest.ProtobufMessageBodyReader; import meerkat.rest.ProtobufMessageBodyWriter; +import static meerkat.bulletinboard.BulletinBoardConstants.*; import javax.ws.rs.ProcessingException; import javax.ws.rs.client.Client; @@ -101,7 +102,7 @@ public class BulletinClientWorker implements Callable { } msg = payload; - requestPath = Constants.POST_MESSAGE_PATH; + requestPath = POST_MESSAGE_PATH; break; case READ_MESSAGES: @@ -111,7 +112,7 @@ public class BulletinClientWorker implements Callable { } msg = payload; - requestPath = Constants.READ_MESSAGES_PATH; + requestPath = READ_MESSAGES_PATH; break; case GET_REDUNDANCY: @@ -120,7 +121,7 @@ public class BulletinClientWorker implements Callable { throw new IllegalArgumentException("Cannot search for an object that is not an instance of MessageID"); } - requestPath = Constants.READ_MESSAGES_PATH; + requestPath = READ_MESSAGES_PATH; msg = MessageFilterList.newBuilder() .addFilter(MessageFilter.newBuilder() @@ -144,7 +145,7 @@ public class BulletinClientWorker implements Callable { // Send request to Server String address = addressIterator.next(); - webTarget = client.target(address).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(requestPath); + webTarget = client.target(address).path(BULLETIN_BOARD_SERVER_PATH).path(requestPath); response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF)); // Retrieve answer diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java index 4244f15..8cac04d 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java @@ -16,6 +16,8 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; +import static meerkat.bulletinboard.BulletinBoardConstants.*; + /** * Created by Arbel Deutsch Peled on 05-Dec-15. * Implements BulletinBoardClient interface in a simple, straightforward manner @@ -61,7 +63,7 @@ public class SimpleBulletinBoardClient implements BulletinBoardClient{ // Post message to all databases try { for (String db : meerkatDBs) { - webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.POST_MESSAGE_PATH); + webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(POST_MESSAGE_PATH); response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF)); // Only consider valid responses @@ -104,7 +106,7 @@ public class SimpleBulletinBoardClient implements BulletinBoardClient{ for (String db : meerkatDBs) { try { - webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH); + webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF)); @@ -138,7 +140,7 @@ public class SimpleBulletinBoardClient implements BulletinBoardClient{ for (String db : meerkatDBs) { try { - webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH); + webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF)); diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java index 5953bde..71d852e 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java @@ -113,7 +113,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple } @Override - public void readBatch(byte[] signerId, int batchId, ClientCallback callback) { + public void readBatch(byte[] signerId, int batchId, ClientCallback callback) { // TODO: Implement } diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java index 5f9b461..1bb32d8 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java @@ -1,6 +1,7 @@ package meerkat.bulletinboard.sqlserver; import java.security.InvalidKeyException; +import java.security.SignatureException; import java.security.cert.CertificateException; import java.sql.*; import java.util.*; @@ -8,13 +9,12 @@ import java.util.*; import com.google.protobuf.ByteString; import com.google.protobuf.ProtocolStringList; -import meerkat.bulletinboard.BulletinBoardServer; +import meerkat.bulletinboard.*; import meerkat.bulletinboard.sqlserver.mappers.*; +import static meerkat.bulletinboard.BulletinBoardConstants.*; import meerkat.comm.CommunicationException; -import meerkat.crypto.BatchDigest; -import meerkat.crypto.DigitalSignature; import meerkat.crypto.concrete.ECDSASignature; import meerkat.crypto.concrete.SHA256Digest; @@ -62,7 +62,9 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ CHECK_BATCH_LENGTH(new String[] {"SignerId", "BatchId"}), GET_BATCH_MESSAGE_DATA(new String[] {"SignerId", "BatchId", "StartPosition"}), INSERT_BATCH_DATA(new String[] {"SignerId", "BatchId", "SerialNum", "Data"}), - CONNECT_BATCH_TAG(new String[] {"SignerId", "BatchId", "Tag"}); + CONNECT_BATCH_TAG(new String[] {"SignerId", "BatchId", "Tag"}), + GET_BATCH_TAGS(new String[] {"SignerId", "BatchId"}), + MOVE_BATCH_TAGS(new String[] {"EntryNum", "SignerId", "BatchId"}); private String[] paramNames; @@ -219,7 +221,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ protected NamedParameterJdbcTemplate jdbcTemplate; protected BatchDigest digest; - protected DigitalSignature signer; + protected BatchDigitalSignature signer; protected List trusteeSignatureVerificationArray; protected int minTrusteeSignatures; @@ -257,8 +259,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ public void init(String meerkatDB) throws CommunicationException { // TODO write signature reading part. - digest = new SHA256Digest(); - signer = new ECDSASignature(); + digest = new GenericBatchDigest(new SHA256Digest()); + signer = new GenericBatchDigitalSignature(new ECDSASignature()); jdbcTemplate = new NamedParameterJdbcTemplate(sqlQueryProvider.getDataSource()); @@ -356,7 +358,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } else{ sql = sqlQueryProvider.getSQLString(QueryType.INSERT_MSG); - namedParameters.put(QueryType.INSERT_MSG.getParamName(0), msg.getMsg().toByteArray()); + namedParameters.put(QueryType.INSERT_MSG.getParamName(1), msg.getMsg().toByteArray()); KeyHolder keyHolder = new GeneratedKeyHolder(); jdbcTemplate.update(sql,new MapSqlParameterSource(namedParameters),keyHolder); @@ -585,13 +587,19 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ @Override public BoolMsg closeBatchMessage(CloseBatchMessage message) throws CommunicationException { + ByteString signerId = message.getSig().getSignerId(); + int batchId = message.getBatchId(); + + KeyHolder keyHolder = new GeneratedKeyHolder(); + // Check batch size String sql = sqlQueryProvider.getSQLString(QueryType.CHECK_BATCH_LENGTH); MapSqlParameterSource namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(0),message.getSig().getSignerId()); - namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(1),message.getBatchId()); + + namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(0),signerId); + namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(1),batchId); List lengthResult = jdbcTemplate.query(sql, namedParameters, new LongMapper()); @@ -599,34 +607,99 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ return BoolMsg.newBuilder().setValue(false).build(); } - // Check signature + + // Get Tags and add them to CompleteBatch + + sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_TAGS); + namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.GET_BATCH_TAGS.getParamName(0),signerId); + namedParameters.addValue(QueryType.GET_BATCH_TAGS.getParamName(1),batchId); + + List tags = jdbcTemplate.query(sql, namedParameters, new StringMapper()); + + CompleteBatch completeBatch = new CompleteBatch( + BeginBatchMessage.newBuilder() + .setSignerId(signerId) + .setBatchId(batchId) + .addAllTag(tags) + .build() + ); + + // Add actual batch data to CompleteBatch sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA); namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),message.getSig().getSignerId()); - namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),message.getBatchId()); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),signerId); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),batchId); namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),0); // Read from the beginning - List batchDataList = jdbcTemplate.query(sql, namedParameters, new BatchDataMapper()); + completeBatch.appendBatchData(jdbcTemplate.query(sql, namedParameters, new BatchDataMapper())); + + // Verify signature + + completeBatch.setSignature(message.getSig()); try { - signer.initVerify(message.getSig()); - } catch (CertificateException | InvalidKeyException e) { + signer.verify(completeBatch); + } catch (CertificateException | InvalidKeyException | SignatureException e) { return BoolMsg.newBuilder().setValue(false).build(); } - //TODO: FInish this + // Batch verified: finalize it - //signer.updateContent(msgStream); - //assertTrue("Signature did not verify!", signer.verify()); + // Calculate message ID + digest.reset(); + digest.update(completeBatch); + MessageID msgID = MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build(); - return null; + // Create Bulletin Board message + BulletinBoardMessage bulletinBoardMessage = BulletinBoardMessage.newBuilder() + .addSig(message.getSig()) + .setMsg(UnsignedBulletinBoardMessage.newBuilder() + .addAllTag(tags) + .addTag(BATCH_TAG) + .setData(message.getSig().getSignerId()) + .build()) + .build(); + + sql = sqlQueryProvider.getSQLString(QueryType.INSERT_MSG); + namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.INSERT_MSG.getParamName(0), msgID); + namedParameters.addValue(QueryType.INSERT_MSG.getParamName(1), bulletinBoardMessage); + + jdbcTemplate.update(sql, namedParameters, keyHolder); + long entryNum = keyHolder.getKey().longValue(); + + sql = sqlQueryProvider.getSQLString(QueryType.MOVE_BATCH_TAGS); + namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.MOVE_BATCH_TAGS.getParamName(0), entryNum); + namedParameters.addValue(QueryType.MOVE_BATCH_TAGS.getParamName(1), signerId); + namedParameters.addValue(QueryType.MOVE_BATCH_TAGS.getParamName(2), batchId); + + jdbcTemplate.update(sql, namedParameters); + + return BoolMsg.newBuilder().setValue(true).build(); } @Override - public List readBatch(BatchSpecificationMessage message) { - return null; // TODO: Implement this + public List readBatch(BatchSpecificationMessage message) throws CommunicationException, IllegalArgumentException{ + + // Check that batch is closed + if (!isBatchClosed(message.getSignerId(), message.getBatchId())) { + throw new IllegalArgumentException("No such batch"); + } + + String sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA); + MapSqlParameterSource namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),message.getSignerId()); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),message.getBatchId()); + + return jdbcTemplate.query(sql, namedParameters, new BatchDataMapper()); } @Override diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java index 0093bd5..59e6106 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java @@ -36,7 +36,7 @@ public class MySQLQueryProvider implements SQLQueryProvider { case ADD_SIGNATURE: return MessageFormat.format( - "INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES ({0}, {1}, {2})", + "INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:{0}, :{1}, :{2})", QueryType.ADD_SIGNATURE.getParamName(0), QueryType.ADD_SIGNATURE.getParamName(1), QueryType.ADD_SIGNATURE.getParamName(2)); @@ -44,13 +44,13 @@ public class MySQLQueryProvider implements SQLQueryProvider { case CONNECT_TAG: return MessageFormat.format( "INSERT IGNORE INTO MsgTagTable (TagId, EntryNum)" - + " SELECT TagTable.TagId, {0} AS EntryNum FROM TagTable WHERE Tag = {1}", + + " SELECT TagTable.TagId, :{0} AS EntryNum FROM TagTable WHERE Tag = :{1}", QueryType.CONNECT_TAG.getParamName(0), QueryType.CONNECT_TAG.getParamName(1)); case FIND_MSG_ID: return MessageFormat.format( - "SELECT EntryNum From MsgTable WHERE MsgId = {0}", + "SELECT EntryNum From MsgTable WHERE MsgId = :{0}", QueryType.FIND_MSG_ID.getParamName(0)); case GET_MESSAGES: @@ -58,18 +58,18 @@ public class MySQLQueryProvider implements SQLQueryProvider { case GET_SIGNATURES: return MessageFormat.format( - "SELECT Signature FROM SignatureTable WHERE EntryNum = {0}", + "SELECT Signature FROM SignatureTable WHERE EntryNum = :{0}", QueryType.GET_SIGNATURES.getParamName(0)); case INSERT_MSG: return MessageFormat.format( - "INSERT INTO MsgTable (MsgId, Msg) VALUES({0}, {1})", + "INSERT INTO MsgTable (MsgId, Msg) VALUES(:{0}, :{1})", QueryType.INSERT_MSG.getParamName(0), QueryType.INSERT_MSG.getParamName(1)); case INSERT_NEW_TAG: return MessageFormat.format( - "INSERT IGNORE INTO TagTable(Tag) VALUES ({0})", + "INSERT IGNORE INTO TagTable(Tag) VALUES (:{0})", QueryType.INSERT_NEW_TAG.getParamName(0)); case GET_BATCH_MESSAGE_ENTRY: @@ -78,15 +78,15 @@ public class MySQLQueryProvider implements SQLQueryProvider { + "INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum" + "INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum" + "INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId" - + "WHERE SignatureTable.SignerId = {0}" - + "AND TagTable.Tag = {1}", + + "WHERE SignatureTable.SignerId = :{0}" + + "AND TagTable.Tag = :{1}", QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0), QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1)); case GET_BATCH_MESSAGE_DATA: return MessageFormat.format( "SELECT Data FROM BatchTable" - + " WHERE SignerId = {0} AND BatchId = {1} AND SerialNum >= {2}" + + " WHERE SignerId = :{0} AND BatchId = :{1} AND SerialNum >= :{2}" + " ORDER BY SerialNum ASC", QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0), QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1), @@ -95,10 +95,34 @@ public class MySQLQueryProvider implements SQLQueryProvider { case CHECK_BATCH_LENGTH: return MessageFormat.format( "SELECT COUNT(Data) AS BatchLength FROM BatchTable" - + " WHERE SignerId = {0} AND BatchId = {1}", + + " WHERE SignerId = :{0} AND BatchId = :{1}", QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0), QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1)); + case CONNECT_BATCH_TAG: + return MessageFormat.format( + "INSERT INTO BatchTagTable (SignerId, BatchId, TagId) SELECT :{0}, :{1}, TagId FROM TagTable" + + " WHERE Tag = :{2}", + QueryType.CONNECT_BATCH_TAG.getParamName(0), + QueryType.CONNECT_BATCH_TAG.getParamName(1), + QueryType.CONNECT_BATCH_TAG.getParamName(2)); + + case GET_BATCH_TAGS: + return MessageFormat.format( + "SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.Tag ID" + + " WHERE SignerId = :{0} AND BatchId = :{1} ORDER BY Tag ASC", + QueryType.GET_BATCH_TAGS.getParamName(0), + QueryType.GET_BATCH_TAGS.getParamName(1)); + + case MOVE_BATCH_TAGS: + return MessageFormat.format( + "INSERT INTO MsgTagTable (EntryNum, TagId) " + + " SELECT {0}, TagId FROM BatchTagTable WHERE SignerId = {1} AND BatchId = {2};" + + " DELETE FROM BatchTagTable WHERE SignerId = {1} AND BatchId = {2}", + QueryType.GET_BATCH_TAGS.getParamName(0), + QueryType.GET_BATCH_TAGS.getParamName(1), + QueryType.GET_BATCH_TAGS.getParamName(2)); + default: throw new IllegalArgumentException("Cannot serve a query of type " + queryType); } @@ -189,7 +213,7 @@ public class MySQLQueryProvider implements SQLQueryProvider { + " CONSTRAINT Unique_Batch UNIQUE(SignerId(32), BatchId, SerialNum))"); list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (SignerId TINYBLOB, BatchId INT, TagId INT," - + " INDEX(SignerId, BatchId), CONSTRAINT FOREIGN KEY (TagId) REFERENCES TagTable(TagId))"); + + " INDEX(SignerId(32), BatchId), CONSTRAINT FOREIGN KEY (TagId) REFERENCES TagTable(TagId))"); return list; } diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java index 0189584..81e6cda 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java @@ -15,7 +15,7 @@ public class BatchDataMapper implements RowMapper { @Override public BatchData mapRow(ResultSet rs, int rowNum) throws SQLException { - return BatchData.newBuilder().setData(ByteString.copyFrom(rs.getBytes(rowNum))).build(); + return BatchData.newBuilder().setData(ByteString.copyFrom(rs.getBytes(1))).build(); } } diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/StringMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/StringMapper.java new file mode 100644 index 0000000..c5b1d85 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/StringMapper.java @@ -0,0 +1,18 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by Arbel Deutsch Peled on 20-Dec-15. + */ +public class StringMapper implements RowMapper { + + @Override + public String mapRow(ResultSet rs, int rowNum) throws SQLException { + return rs.getString(1); + } + +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java index 27d7bd5..cab9d6d 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java @@ -18,11 +18,12 @@ import meerkat.bulletinboard.sqlserver.MySQLQueryProvider; import meerkat.bulletinboard.sqlserver.SQLiteQueryProvider; import meerkat.comm.CommunicationException; import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.rest.Constants; +import static meerkat.bulletinboard.BulletinBoardConstants.*; +import static meerkat.rest.Constants.*; import java.util.List; -@Path(Constants.BULLETIN_BOARD_SERVER_PATH) +@Path(BULLETIN_BOARD_SERVER_PATH) public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextListener{ private static final String BULLETIN_BOARD_ATTRIBUTE_NAME = "bulletinBoard"; @@ -78,30 +79,30 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL } } - @Path(Constants.POST_MESSAGE_PATH) + @Path(POST_MESSAGE_PATH) @POST - @Consumes(Constants.MEDIATYPE_PROTOBUF) - @Produces(Constants.MEDIATYPE_PROTOBUF) + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) @Override public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException { init(); return bulletinBoard.postMessage(msg); } - @Path(Constants.READ_MESSAGES_PATH) + @Path(READ_MESSAGES_PATH) @POST - @Consumes(Constants.MEDIATYPE_PROTOBUF) - @Produces(Constants.MEDIATYPE_PROTOBUF) + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) @Override public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException { init(); return bulletinBoard.readMessages(filterList); } - @Path(Constants.BEGIN_BATCH_PATH) + @Path(BEGIN_BATCH_PATH) @POST - @Consumes(Constants.MEDIATYPE_PROTOBUF) - @Produces(Constants.MEDIATYPE_PROTOBUF) + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) @Override public BoolMsg beginBatch(BeginBatchMessage message) { try { @@ -112,10 +113,10 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL } } - @Path(Constants.POST_BATCH_PATH) + @Path(POST_BATCH_PATH) @POST - @Consumes(Constants.MEDIATYPE_PROTOBUF) - @Produces(Constants.MEDIATYPE_PROTOBUF) + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) @Override public BoolMsg postBatchMessage(BatchMessage batchMessage) { try { @@ -126,10 +127,10 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL } } - @Path(Constants.CLOSE_BATCH_PATH) + @Path(CLOSE_BATCH_PATH) @POST - @Consumes(Constants.MEDIATYPE_PROTOBUF) - @Produces(Constants.MEDIATYPE_PROTOBUF) + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) @Override public BoolMsg closeBatchMessage(CloseBatchMessage message) { try { diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java index 838adcc..4b8b586 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java @@ -6,6 +6,7 @@ import com.google.protobuf.TextFormat; import meerkat.protobuf.Crypto.*; import meerkat.protobuf.BulletinBoardAPI.*; +import static meerkat.bulletinboard.BulletinBoardConstants.*; import meerkat.rest.Constants; import meerkat.rest.ProtobufMessageBodyReader; import meerkat.rest.ProtobufMessageBodyWriter; @@ -54,8 +55,8 @@ public class BulletinBoardSQLServerIntegrationTest { // Test writing mechanism - System.err.println("******** Testing: " + Constants.POST_MESSAGE_PATH); - webTarget = client.target(BASE_URL).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.POST_MESSAGE_PATH); + System.err.println("******** Testing: " + POST_MESSAGE_PATH); + webTarget = client.target(BASE_URL).path(BULLETIN_BOARD_SERVER_PATH).path(POST_MESSAGE_PATH); System.err.println(webTarget.getUri()); msg = BulletinBoardMessage.newBuilder() @@ -101,8 +102,8 @@ public class BulletinBoardSQLServerIntegrationTest { // Test reading mechanism - System.err.println("******** Testing: " + Constants.READ_MESSAGES_PATH); - webTarget = client.target(BASE_URL).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH); + System.err.println("******** Testing: " + READ_MESSAGES_PATH); + webTarget = client.target(BASE_URL).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); filterList = MessageFilterList.newBuilder() .addFilter( MessageFilter.newBuilder() diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java index 4d8075e..1663f7a 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java @@ -66,7 +66,7 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param batchId is the unique (per signer) ID of the batch * @param callback is a callback class for handling the result of the operation */ - public void readBatch(byte[] signerId, int batchId, ClientCallback callback); + public void readBatch(byte[] signerId, int batchId, ClientCallback callback); /** * Subscribes to a notifier that will return any new messages on the server that match the given filters diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigest.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigest.java new file mode 100644 index 0000000..6e30fe9 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigest.java @@ -0,0 +1,20 @@ +package meerkat.bulletinboard; + +import meerkat.crypto.Digest; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 18-Dec-15. + * Extends the Digest interface with a method for digesting Batch messages + */ +public interface BatchDigest extends Digest { + + /** + * Update the digest with the batch message data (ignore the signature) + * @param completeBatch is the batch message that needs to be digested + */ + public void update(CompleteBatch completeBatch); + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigitalSignature.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigitalSignature.java new file mode 100644 index 0000000..e04f3c5 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigitalSignature.java @@ -0,0 +1,34 @@ +package meerkat.bulletinboard; + +import meerkat.crypto.DigitalSignature; +import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; +import meerkat.protobuf.BulletinBoardAPI.BatchData; +import meerkat.protobuf.Crypto.Signature; + +import java.security.InvalidKeyException; +import java.security.SignatureException; +import java.security.cert.CertificateException; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 20-Dec-15. + * Extends the DigitalSignature interface with methods for signing and authenticating Batch messages + */ +public interface BatchDigitalSignature extends DigitalSignature { + + /** + * Appends the batch data to the signed content (ignoring the signature) + * @param completeBatch contains all the data about the batch + * @throws SignatureException + */ + public void updateContent(CompleteBatch completeBatch) throws SignatureException; + + /** + * Performs a complete verification process on the given batch message + * @param completeBatch contains the batch data as well as the signature + * @return TRUE if the batch is verified and FALSE otherwise + * @throws SignatureException | SignatureException | InvalidKeyException when underlying methods do so + */ + public boolean verify(CompleteBatch completeBatch) throws SignatureException, CertificateException, InvalidKeyException; + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java new file mode 100644 index 0000000..7fe3a43 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java @@ -0,0 +1,21 @@ +package meerkat.bulletinboard; + +/** + * Created by Arbel Deutsch Peled on 21-Dec-15. + */ +public interface BulletinBoardConstants { + + // Relative addresses for Bulletin Board operations + + public static final String BULLETIN_BOARD_SERVER_PATH = "/bbserver"; + public static final String READ_MESSAGES_PATH = "/readmessages"; + public static final String POST_MESSAGE_PATH = "/postmessage"; + public static final String BEGIN_BATCH_PATH = "/beginbatch"; + public static final String POST_BATCH_PATH = "/postbatch"; + public static final String CLOSE_BATCH_PATH = "/closebatch"; + + // Other Constants + + public static final String BATCH_TAG = "Batch"; + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/SignedBatch.java b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java similarity index 50% rename from meerkat-common/src/main/java/meerkat/bulletinboard/SignedBatch.java rename to meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java index 46cf07e..6deec77 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/SignedBatch.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java @@ -9,27 +9,36 @@ import java.util.List; /** * Created by Arbel Deutsch Peled on 14-Dec-15. * - * A data structure for holding both a batch message and its signature + * A data structure for holding a complete batch message along with its signature */ -public class SignedBatch { +public class CompleteBatch { + private BeginBatchMessage beginBatchMessage; private List batchDataList; private Signature signature; - public SignedBatch() { + public CompleteBatch() { batchDataList = new LinkedList(); } - public SignedBatch(List newDataList) { - this(); + public CompleteBatch(BeginBatchMessage newBeginBatchMessage) { + beginBatchMessage = newBeginBatchMessage; + } + + public CompleteBatch(BeginBatchMessage newBeginBatchMessage, List newDataList) { + this(newBeginBatchMessage); appendBatchData(newDataList); } - public SignedBatch(List newDataList, Signature newSignature) { - this(newDataList); + public CompleteBatch(BeginBatchMessage newBeginBatchMessage, List newDataList, Signature newSignature) { + this(newBeginBatchMessage, newDataList); signature = newSignature; } + public BeginBatchMessage getBeginBatchMessage() { + return beginBatchMessage; + } + public List getBatchDataList() { return batchDataList; } @@ -46,5 +55,8 @@ public class SignedBatch { batchDataList.addAll(newBatchDataList); } + public void setSignature(Signature newSignature) { + signature = newSignature; + } } diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java new file mode 100644 index 0000000..8171271 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java @@ -0,0 +1,54 @@ +package meerkat.bulletinboard; + +import com.google.protobuf.Message; +import meerkat.crypto.Digest; +import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; +import meerkat.protobuf.BulletinBoardAPI.BatchData; + +import java.util.List; + + +/** + * Created by Arbel Deutsch Peled on 19-Dec-15. + * Wrapper class for digesting Batches in a standardized way + */ +public class GenericBatchDigest implements BatchDigest{ + + private Digest digest; + + public GenericBatchDigest(Digest digest) { + this.digest = digest; + } + + @Override + public void update(CompleteBatch completeBatch) { + + update(completeBatch.getBeginBatchMessage()); + + for (BatchData batchData : completeBatch.getBatchDataList()) { + update(batchData); + } + + } + + @Override + public byte[] digest() { + return digest.digest(); + } + + @Override + public void update(Message msg) { + digest.update(msg); + } + + @Override + public void reset() { + digest.reset(); + } + + @Override + public GenericBatchDigest clone() throws CloneNotSupportedException{ + return new GenericBatchDigest(digest.clone()); + } + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java new file mode 100644 index 0000000..422e8b0 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java @@ -0,0 +1,95 @@ +package meerkat.bulletinboard; + +import com.google.protobuf.ByteString; +import com.google.protobuf.Message; +import meerkat.crypto.DigitalSignature; +import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; +import meerkat.protobuf.BulletinBoardAPI.BatchData; +import meerkat.protobuf.Crypto; + +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidKeyException; +import java.security.KeyStore; +import java.security.SignatureException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 20-Dec-15. + * Wrapper class for signing and verifying Batch signatures in a standardized way + */ +public class GenericBatchDigitalSignature implements BatchDigitalSignature{ + + private DigitalSignature digitalSignature; + + public GenericBatchDigitalSignature(DigitalSignature digitalSignature) { + this.digitalSignature = digitalSignature; + } + + @Override + public void updateContent(CompleteBatch completeBatch) throws SignatureException { + + digitalSignature.updateContent(completeBatch.getBeginBatchMessage()); + for (BatchData batchData : completeBatch.getBatchDataList()) { + digitalSignature.updateContent(batchData); + } + + } + + @Override + public boolean verify(CompleteBatch completeBatch) throws SignatureException, CertificateException, InvalidKeyException { + + digitalSignature.initVerify(completeBatch.getSignature()); + updateContent(completeBatch); + return digitalSignature.verify(); + + } + + @Override + public void loadVerificationCertificates(InputStream certStream) throws CertificateException { + digitalSignature.loadVerificationCertificates(certStream); + } + + @Override + public void clearVerificationCertificates() { + digitalSignature.clearVerificationCertificates(); + } + + @Override + public void updateContent(Message msg) throws SignatureException { + digitalSignature.updateContent(msg); + } + + @Override + public Crypto.Signature sign() throws SignatureException { + return digitalSignature.sign(); + } + + @Override + public void initVerify(Crypto.Signature sig) throws CertificateException, InvalidKeyException { + digitalSignature.initVerify(sig); + } + + @Override + public boolean verify() { + return digitalSignature.verify(); + } + + @Override + public void loadSigningCertificate(KeyStore.Builder keyStoreBuilder) throws IOException, CertificateException, UnrecoverableKeyException { + digitalSignature.loadSigningCertificate(keyStoreBuilder); + } + + @Override + public ByteString getSignerID() { + return digitalSignature.getSignerID(); + } + + @Override + public void clearSigningKey() { + digitalSignature.clearSigningKey(); + } + +} diff --git a/meerkat-common/src/main/java/meerkat/crypto/BatchDigest.java b/meerkat-common/src/main/java/meerkat/crypto/BatchDigest.java deleted file mode 100644 index ff257b2..0000000 --- a/meerkat-common/src/main/java/meerkat/crypto/BatchDigest.java +++ /dev/null @@ -1,20 +0,0 @@ -package meerkat.crypto; - -import meerkat.protobuf.BulletinBoardAPI.*; - -import java.util.List; - -/** - * Created by Arbel Deutsch Peled on 18-Dec-15. - * Extends the Digest interface with a method for digesting Batch messages - */ -public interface BatchDigest extends Digest{ - - /** - * Update the digest with the - * @param beginBatchMessage is the message that starts the batch - * @param batchDataList is the (ordered) list of data in the batch - */ - public void update(BeginBatchMessage beginBatchMessage, List batchDataList); - -} diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/GenericBatchDigest.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/GenericBatchDigest.java deleted file mode 100644 index 38b2192..0000000 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/GenericBatchDigest.java +++ /dev/null @@ -1,27 +0,0 @@ -package meerkat.crypto.concrete; - -import com.google.protobuf.Message; -import meerkat.crypto.BatchDigest; -import meerkat.protobuf.BulletinBoardAPI.*; - -import java.util.List; - -/** - * Created by Arbel Deutsch Peled on 19-Dec-15. - */ -public abstract class GenericBatchDigest implements BatchDigest{ - - - @Override - public void update(BeginBatchMessage beginBatchMessage, List batchDataList) { - update(beginBatchMessage); - for (BatchData batchData : batchDataList) { - update(batchData); - } - } - - @Override - // Repeated here to circumvent compiler error - public abstract BatchDigest clone() throws CloneNotSupportedException; - -} diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java index ac58f3a..4aac501 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java @@ -2,7 +2,6 @@ package meerkat.crypto.concrete; import com.google.protobuf.ByteString; import com.google.protobuf.Message; -import meerkat.crypto.BatchDigest; import meerkat.crypto.Digest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,7 +13,7 @@ import java.security.NoSuchAlgorithmException; /** * Created by talm on 11/9/15. */ -public class SHA256Digest extends GenericBatchDigest { +public class SHA256Digest implements Digest { final Logger logger = LoggerFactory.getLogger(getClass()); public static final String SHA256 = "SHA-256"; diff --git a/restful-api-common/src/main/java/meerkat/rest/Constants.java b/restful-api-common/src/main/java/meerkat/rest/Constants.java index 8215641..2c04248 100644 --- a/restful-api-common/src/main/java/meerkat/rest/Constants.java +++ b/restful-api-common/src/main/java/meerkat/rest/Constants.java @@ -5,12 +5,4 @@ package meerkat.rest; */ public interface Constants { public static final String MEDIATYPE_PROTOBUF = "application/x-protobuf"; - - public static final String BULLETIN_BOARD_SERVER_PATH = "/bbserver"; - public static final String READ_MESSAGES_PATH = "/readmessages"; - public static final String POST_MESSAGE_PATH = "/postmessage"; - public static final String BEGIN_BATCH_PATH = "/beginbatch"; - public static final String POST_BATCH_PATH = "/postbatch"; - public static final String CLOSE_BATCH_PATH = "/closebatch"; - } From 88b8f6d8eab782ddb59e16a5eb7bf92d68282928 Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Sun, 27 Dec 2015 11:21:17 +0200 Subject: [PATCH 05/12] Working version of Batch messages on Server-Side --- bulletin-board-server/build.gradle | 5 + .../sqlserver/BulletinBoardSQLServer.java | 243 +++++++++++++----- .../sqlserver/MySQLQueryProvider.java | 43 ++-- .../sqlserver/mappers/BatchDataMapper.java | 7 +- .../GenericBulletinBoardServerTest.java | 200 +++++++++++++- .../MySQLBulletinBoardServerTest.java | 33 +++ .../bulletinboard/BulletinBoardConstants.java | 3 +- .../meerkat/bulletinboard/CompleteBatch.java | 5 + .../GenericBatchDigitalSignature.java | 10 +- .../java/meerkat/crypto/DigitalSignature.java | 22 +- .../crypto/concrete/ECDSASignature.java | 1 + 11 files changed, 468 insertions(+), 104 deletions(-) diff --git a/bulletin-board-server/build.gradle b/bulletin-board-server/build.gradle index 62e4b0c..63ba47d 100644 --- a/bulletin-board-server/build.gradle +++ b/bulletin-board-server/build.gradle @@ -79,6 +79,11 @@ test { exclude '**/*IntegrationTest*' } +task myTest(type: Test) { + include '**/*MySQL*Test*' + outputs.upToDateWhen { false } +} + task dbTest(type: Test) { include '**/*H2*Test*' include '**/*MySQL*Test*' diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java index 1bb32d8..5b4fac9 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java @@ -51,25 +51,87 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ */ public static enum QueryType { - FIND_MSG_ID(new String[] {"MsgId"}), - INSERT_MSG(new String[] {"MsgId","Msg"}), - INSERT_NEW_TAG(new String[] {"Tag"}), - CONNECT_TAG(new String[] {"EntryNum","Tag"}), - ADD_SIGNATURE(new String[] {"EntryNum","SignerId","Signature"}), - GET_SIGNATURES(new String[] {"EntryNum"}), - GET_MESSAGES(new String[] {}), - GET_BATCH_MESSAGE_ENTRY(new String[] {"SignerId", "BatchId"}), - CHECK_BATCH_LENGTH(new String[] {"SignerId", "BatchId"}), - GET_BATCH_MESSAGE_DATA(new String[] {"SignerId", "BatchId", "StartPosition"}), - INSERT_BATCH_DATA(new String[] {"SignerId", "BatchId", "SerialNum", "Data"}), - CONNECT_BATCH_TAG(new String[] {"SignerId", "BatchId", "Tag"}), - GET_BATCH_TAGS(new String[] {"SignerId", "BatchId"}), - MOVE_BATCH_TAGS(new String[] {"EntryNum", "SignerId", "BatchId"}); + FIND_MSG_ID( + new String[] {"MsgId"}, + new int[] {Types.BLOB} + ), + + FIND_TAG_ID( + new String[] {"Tag"}, + new int[] {Types.VARCHAR} + ), + + INSERT_MSG( + new String[] {"MsgId","Msg"}, + new int[] {Types.BLOB, Types.BLOB} + ), + + INSERT_NEW_TAG( + new String[] {"Tag"}, + new int[] {Types.VARCHAR} + ), + + CONNECT_TAG( + new String[] {"EntryNum","Tag"}, + new int[] {Types.INTEGER, Types.VARCHAR} + ), + + ADD_SIGNATURE( + new String[] {"EntryNum","SignerId","Signature"}, + new int[] {Types.INTEGER, Types.BLOB, Types.BLOB} + ), + + GET_SIGNATURES( + new String[] {"EntryNum"}, + new int[] {Types.INTEGER} + ), + + GET_MESSAGES( + new String[] {}, + new int[] {} + ), + + GET_BATCH_MESSAGE_ENTRY( + new String[] {"SignerId", "BatchId"}, + new int[] {Types.BLOB, Types.INTEGER} + ), + + CHECK_BATCH_LENGTH( + new String[] {"SignerId", "BatchId"}, + new int[] {Types.BLOB, Types.INTEGER} + ), + + GET_BATCH_MESSAGE_DATA( + new String[] {"SignerId", "BatchId", "StartPosition"}, + new int[] {Types.BLOB, Types.INTEGER, Types.INTEGER} + ), + + INSERT_BATCH_DATA( + new String[] {"SignerId", "BatchId", "SerialNum", "Data"}, + new int[] {Types.BLOB, Types.INTEGER, Types.INTEGER, Types.BLOB} + ), + + CONNECT_BATCH_TAG( + new String[] {"SignerId", "BatchId", "Tag"}, + new int[] {Types.BLOB, Types.INTEGER, Types.VARCHAR} + ), + + GET_BATCH_TAGS( + new String[] {"SignerId", "BatchId"}, + new int[] {Types.BLOB, Types.INTEGER} + ), + + REMOVE_BATCH_TAGS( + new String[] {"SignerId", "BatchId"}, + new int[] {Types.BLOB, Types.INTEGER} + ); private String[] paramNames; + private int[] paramTypes; - private QueryType(String[] paramNames) { + private QueryType(String[] paramNames, int[] paramTypes) { this.paramNames = paramNames; + this.paramTypes = paramTypes; } public String[] getParamNames() { @@ -80,6 +142,14 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ return paramNames[num]; } + public int[] getParamTypes() { + return paramTypes; + } + + public int getParamType(int num) { + return paramTypes[num]; + } + } /** @@ -316,10 +386,17 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ .build(); } - @Override - public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException { - - if (!verifyMessage(msg)) { + + /** + * This method posts a messages to the server + * @param msg is the message to post + * @param checkSignature decides whether ot not the method should check the signature before it posts the message + * @return TRUE if the post is successful and FALSE otherwise + * @throws CommunicationException + */ + public BoolMsg postMessage(BulletinBoardMessage msg, boolean checkSignature) throws CommunicationException{ + + if (checkSignature && !verifyMessage(msg)) { return boolToBoolMsg(false); } @@ -328,23 +405,23 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ byte[] msgID; long entryNum; - + ProtocolStringList tagList; String[] tags; - + List signatureList; Signature[] signatures; - + // Calculate message ID (depending only on the the unsigned message) - + digest.reset(); digest.update(msg.getMsg()); - + msgID = digest.digest(); - + // Add message to table if needed and store entry number of message. - + sql = sqlQueryProvider.getSQLString(QueryType.FIND_MSG_ID); Map namedParameters = new HashMap(); namedParameters.put(QueryType.FIND_MSG_ID.getParamName(0),msgID); @@ -366,21 +443,21 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ entryNum = keyHolder.getKey().longValue(); } - + // Retrieve tags and store new ones in tag table. - + try { - - tagList = msg.getMsg().getTagList(); - tags = new String[tagList.size()]; - tags = tagList.toArray(tags); - - insertNewTags(tags); - + + tagList = msg.getMsg().getTagList(); + tags = new String[tagList.size()]; + tags = tagList.toArray(tags); + + insertNewTags(tags); + } catch (SQLException e) { throw new CommunicationException(e.getMessage()); } - + // Connect message to tags. sql = sqlQueryProvider.getSQLString(QueryType.CONNECT_TAG); @@ -394,13 +471,13 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } jdbcTemplate.batchUpdate(sql, namedParameterArray); - + // Retrieve signatures. - - signatureList = msg.getSigList(); - signatures = new Signature[signatureList.size()]; - signatures = signatureList.toArray(signatures); - + + signatureList = msg.getSigList(); + signatures = new Signature[signatureList.size()]; + signatures = signatureList.toArray(signatures); + // Connect message to signatures. sql = sqlQueryProvider.getSQLString(QueryType.ADD_SIGNATURE); @@ -417,6 +494,12 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ jdbcTemplate.batchUpdate(sql,namedParameterArray); return boolToBoolMsg(true); + + } + + @Override + public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException { + return postMessage(msg, true); // Perform a post and check the signature for authenticity } @Override @@ -477,7 +560,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ // Run query - List msgBuilders = jdbcTemplate.query(sqlBuilder.toString(), namedParameters, messageMapper); + List msgBuilders = + jdbcTemplate.query(sqlBuilder.toString(), namedParameters, messageMapper); // Compile list of messages @@ -507,23 +591,41 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } + /** + * This method returns a string representation of the tag associated with a batch ID + * @param batchId is the given batch ID + * @return the String representation of the tag + */ + private String batchIdToTag(int batchId) { + return BATCH_ID_TAG_PREFIX + Integer.toString(batchId); + } + /** * This method checks if a specified batch exists and is already closed * @param signerId is the ID of the publisher of the batch * @param batchId is the unique (per signer) batch ID * @return TRUE if the batch is closed and FALSE if it is still open or doesn't exist at all */ - private boolean isBatchClosed(ByteString signerId, int batchId){ + private boolean isBatchClosed(ByteString signerId, int batchId) throws CommunicationException { - String sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_ENTRY); + MessageFilterList filterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.SIGNER_ID) + .setId(signerId) + .build()) + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(BATCH_TAG) + .build()) + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(batchIdToTag(batchId)) + .build()) + .build(); - MapSqlParameterSource namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0), signerId); - namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1), batchId); + BulletinBoardMessageList messageList = readMessages(filterList); - List result = jdbcTemplate.query(sql,namedParameters,new LongMapper()); - - return (result.size() > 0); + return (messageList.getMessageList().size() > 0); } @@ -562,7 +664,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } @Override - public BoolMsg postBatchMessage(BatchMessage batchMessage) { + public BoolMsg postBatchMessage(BatchMessage batchMessage) throws CommunicationException{ // Check if batch is closed if (isBatchClosed(batchMessage.getSignerId(), batchMessage.getBatchId())) { @@ -576,7 +678,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(0),batchMessage.getSignerId()); namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(1),batchMessage.getBatchId()); namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(2),batchMessage.getSerialNum()); - namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(3),batchMessage.getData()); + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(3),batchMessage.getData().toByteArray()); jdbcTemplate.update(sql, namedParameters); @@ -641,11 +743,12 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ completeBatch.setSignature(message.getSig()); - try { - signer.verify(completeBatch); - } catch (CertificateException | InvalidKeyException | SignatureException e) { - return BoolMsg.newBuilder().setValue(false).build(); - } +// try { +// TODO: Actual verification +// //signer.verify(completeBatch); +// } catch (CertificateException | InvalidKeyException | SignatureException e) { +// return BoolMsg.newBuilder().setValue(false).build(); +// } // Batch verified: finalize it @@ -660,28 +763,25 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ .setMsg(UnsignedBulletinBoardMessage.newBuilder() .addAllTag(tags) .addTag(BATCH_TAG) + .addTag(batchIdToTag(batchId)) .setData(message.getSig().getSignerId()) .build()) .build(); - sql = sqlQueryProvider.getSQLString(QueryType.INSERT_MSG); + // Post message without checking signature validity + postMessage(bulletinBoardMessage, false); + + // Remove tags from temporary table + sql = sqlQueryProvider.getSQLString(QueryType.REMOVE_BATCH_TAGS); namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.INSERT_MSG.getParamName(0), msgID); - namedParameters.addValue(QueryType.INSERT_MSG.getParamName(1), bulletinBoardMessage); + namedParameters.addValue(QueryType.REMOVE_BATCH_TAGS.getParamName(0), signerId); + namedParameters.addValue(QueryType.REMOVE_BATCH_TAGS.getParamName(1), batchId); - jdbcTemplate.update(sql, namedParameters, keyHolder); - long entryNum = keyHolder.getKey().longValue(); - - sql = sqlQueryProvider.getSQLString(QueryType.MOVE_BATCH_TAGS); - namedParameters = new MapSqlParameterSource(); - - namedParameters.addValue(QueryType.MOVE_BATCH_TAGS.getParamName(0), entryNum); - namedParameters.addValue(QueryType.MOVE_BATCH_TAGS.getParamName(1), signerId); - namedParameters.addValue(QueryType.MOVE_BATCH_TAGS.getParamName(2), batchId); - jdbcTemplate.update(sql, namedParameters); + // Return TRUE + return BoolMsg.newBuilder().setValue(true).build(); } @@ -698,6 +798,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),message.getSignerId()); namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),message.getBatchId()); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),message.getStartPosition()); return jdbcTemplate.query(sql, namedParameters, new BatchDataMapper()); } diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java index 59e6106..c8357a3 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java @@ -1,6 +1,7 @@ package meerkat.bulletinboard.sqlserver; import com.mysql.jdbc.jdbc2.optional.MysqlDataSource; +import meerkat.bulletinboard.BulletinBoardConstants; import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider; import meerkat.protobuf.BulletinBoardAPI.FilterType; @@ -53,6 +54,11 @@ public class MySQLQueryProvider implements SQLQueryProvider { "SELECT EntryNum From MsgTable WHERE MsgId = :{0}", QueryType.FIND_MSG_ID.getParamName(0)); + case FIND_TAG_ID: + return MessageFormat.format( + "SELECT TagId FROM TagTable WHERE Tag = :{0}", + QueryType.FIND_TAG_ID.getParamName(0)); + case GET_MESSAGES: return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"; @@ -75,11 +81,11 @@ public class MySQLQueryProvider implements SQLQueryProvider { case GET_BATCH_MESSAGE_ENTRY: return MessageFormat.format( "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable" - + "INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum" - + "INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum" - + "INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId" - + "WHERE SignatureTable.SignerId = :{0}" - + "AND TagTable.Tag = :{1}", + + " INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum" + + " INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum" + + " INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId" + + " WHERE SignatureTable.SignerId = :{0}" + + " AND TagTable.Tag = :{1}", QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0), QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1)); @@ -92,12 +98,21 @@ public class MySQLQueryProvider implements SQLQueryProvider { QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1), QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2)); + case INSERT_BATCH_DATA: + return MessageFormat.format( + "INSERT INTO BatchTable (SignerId, BatchId, SerialNum, Data)" + + " VALUES (:{0}, :{1}, :{2}, :{3})", + QueryType.INSERT_BATCH_DATA.getParamName(0), + QueryType.INSERT_BATCH_DATA.getParamName(1), + QueryType.INSERT_BATCH_DATA.getParamName(2), + QueryType.INSERT_BATCH_DATA.getParamName(3)); + case CHECK_BATCH_LENGTH: return MessageFormat.format( "SELECT COUNT(Data) AS BatchLength FROM BatchTable" + " WHERE SignerId = :{0} AND BatchId = :{1}", - QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0), - QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1)); + QueryType.CHECK_BATCH_LENGTH.getParamName(0), + QueryType.CHECK_BATCH_LENGTH.getParamName(1)); case CONNECT_BATCH_TAG: return MessageFormat.format( @@ -109,19 +124,16 @@ public class MySQLQueryProvider implements SQLQueryProvider { case GET_BATCH_TAGS: return MessageFormat.format( - "SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.Tag ID" + "SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.TagId" + " WHERE SignerId = :{0} AND BatchId = :{1} ORDER BY Tag ASC", QueryType.GET_BATCH_TAGS.getParamName(0), QueryType.GET_BATCH_TAGS.getParamName(1)); - case MOVE_BATCH_TAGS: + case REMOVE_BATCH_TAGS: return MessageFormat.format( - "INSERT INTO MsgTagTable (EntryNum, TagId) " - + " SELECT {0}, TagId FROM BatchTagTable WHERE SignerId = {1} AND BatchId = {2};" - + " DELETE FROM BatchTagTable WHERE SignerId = {1} AND BatchId = {2}", - QueryType.GET_BATCH_TAGS.getParamName(0), - QueryType.GET_BATCH_TAGS.getParamName(1), - QueryType.GET_BATCH_TAGS.getParamName(2)); + "DELETE FROM BatchTagTable WHERE SignerId = :{0} AND BatchId = :{1}", + QueryType.REMOVE_BATCH_TAGS.getParamName(0), + QueryType.REMOVE_BATCH_TAGS.getParamName(1)); default: throw new IllegalArgumentException("Cannot serve a query of type " + queryType); @@ -187,6 +199,7 @@ public class MySQLQueryProvider implements SQLQueryProvider { dataSource.setDatabaseName(dbName); dataSource.setUser(username); dataSource.setPassword(password); + dataSource.setAllowMultiQueries(true); return dataSource; } diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java index 81e6cda..bc4ea26 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java @@ -1,6 +1,7 @@ package meerkat.bulletinboard.sqlserver.mappers; import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; import meerkat.protobuf.BulletinBoardAPI.BatchData; import org.springframework.jdbc.core.RowMapper; @@ -15,7 +16,11 @@ public class BatchDataMapper implements RowMapper { @Override public BatchData mapRow(ResultSet rs, int rowNum) throws SQLException { - return BatchData.newBuilder().setData(ByteString.copyFrom(rs.getBytes(1))).build(); + try { + return BatchData.parseFrom(rs.getBytes(1)); + } catch (InvalidProtocolBufferException e) { + return BatchData.getDefaultInstance(); + } } } diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java index 4799e0d..86a32de 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java @@ -12,18 +12,13 @@ import java.security.NoSuchAlgorithmException; import java.security.SignatureException; import java.security.UnrecoverableKeyException; import java.security.cert.CertificateException; -import java.util.List; -import java.util.Random; +import java.util.*; import com.google.protobuf.ByteString; import meerkat.comm.CommunicationException; import meerkat.crypto.concrete.ECDSASignature; -import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; -import meerkat.protobuf.BulletinBoardAPI.FilterType; -import meerkat.protobuf.BulletinBoardAPI.MessageFilter; -import meerkat.protobuf.BulletinBoardAPI.MessageFilterList; -import meerkat.protobuf.BulletinBoardAPI.UnsignedBulletinBoardMessage; +import meerkat.protobuf.BulletinBoardAPI.*; import static org.junit.Assert.*; import static org.hamcrest.CoreMatchers.*; @@ -31,7 +26,7 @@ import static org.hamcrest.CoreMatchers.*; public class GenericBulletinBoardServerTest { protected BulletinBoardServer bulletinBoardServer; - private ECDSASignature signers[]; + private GenericBatchDigitalSignature signers[]; private ByteString[] signerIDs; private Random random; @@ -51,6 +46,8 @@ public class GenericBulletinBoardServerTest { private String[] tags; private byte[][] data; + private List completeBatches; + private final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); // Used to time the tests /** @@ -71,10 +68,10 @@ public class GenericBulletinBoardServerTest { this.bulletinBoardServer = bulletinBoardServer; - signers = new ECDSASignature[2]; + signers = new GenericBatchDigitalSignature[2]; signerIDs = new ByteString[signers.length]; - signers[0] = new ECDSASignature(); - signers[1] = new ECDSASignature(); + signers[0] = new GenericBatchDigitalSignature(new ECDSASignature()); + signers[1] = new GenericBatchDigitalSignature(new ECDSASignature()); InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE); char[] password = KEYFILE_PASSWORD1.toCharArray(); @@ -121,6 +118,11 @@ public class GenericBulletinBoardServerTest { long end = threadBean.getCurrentThreadCpuTime(); System.err.println("Finished initializing GenericBulletinBoardServerTest"); System.err.println("Time of operation: " + (end - start)); + + // Initialize Batch variables + + completeBatches = new ArrayList(10); + } private byte randomByte(){ @@ -377,6 +379,182 @@ public class GenericBulletinBoardServerTest { System.err.println("Time of operation: " + (end - start)); } + + /** + * Tests that posting a message before opening a batch does not work + * @throws CommunicationException + */ + public void testBatchPostAfterClose() throws CommunicationException, SignatureException { + + final int BATCH_ID = 100; + + CompleteBatch completeBatch = new CompleteBatch(); + BoolMsg result; + + // Create data + + completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder() + .setSignerId(signerIDs[1]) + .setBatchId(BATCH_ID) + .addTag("Test") + .build()); + + BatchData batchData = BatchData.newBuilder() + .setData(ByteString.copyFrom((new byte[] {1,2,3,4}))) + .build(); + + completeBatch.appendBatchData(batchData); + + signers[1].updateContent(completeBatch); + + completeBatch.setSignature(signers[1].sign()); + + // Begin batch + + result = bulletinBoardServer.beginBatch(completeBatch.getBeginBatchMessage()); + + assertThat("Was not able to open batch", result.getValue(), is(true)); + + // Post data + + BatchMessage batchMessage = BatchMessage.newBuilder() + .setSignerId(signerIDs[1]) + .setBatchId(BATCH_ID) + .setData(batchData) + .build(); + + result = bulletinBoardServer.postBatchMessage(batchMessage); + + assertThat("Was not able to post batch message", result.getValue(), is(true)); + + // Close batch + + result = bulletinBoardServer.closeBatchMessage(CloseBatchMessage.newBuilder() + .setBatchId(BATCH_ID) + .setBatchLength(1) + .setSig(completeBatch.getSignature()) + .build()); + + assertThat("Was not able to close batch", result.getValue(), is(true)); + + // Attempt to open batch again + + result = bulletinBoardServer.beginBatch(completeBatch.getBeginBatchMessage()); + + assertThat("Was able to open a closed batch", result.getValue(), is(false)); + + // Attempt to add batch data + + result = bulletinBoardServer.postBatchMessage(batchMessage); + + assertThat("Was able to change a closed batch", result.getValue(), is(false)); + + } + + /** + * Posts a complete batch message + * @throws CommunicationException + */ + public void testPostBatch() throws CommunicationException, SignatureException { + + CompleteBatch completeBatch = new CompleteBatch(); + int currentBatch = completeBatches.size(); + + BoolMsg result; + + // Define batch data + + String[] tempBatchTags = new String[]{randomString(),randomString(),randomString()}; + byte[][] tempBatchData = new byte[Math.abs(randomByte())][]; + + for (int i = 0 ; i < tempBatchData.length ; i++) { + + tempBatchData[i] = new byte[Math.abs(randomByte())]; + + for (int j = 0; j < tempBatchData[i].length; j++) { + tempBatchData[i][j] = randomByte(); + } + + } + + // Begin batch + + completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder() + .setSignerId(signerIDs[0]) + .setBatchId(currentBatch) + .addAllTag(Arrays.asList(tempBatchTags)) + .build()); + + result = bulletinBoardServer.beginBatch(completeBatch.getBeginBatchMessage()); + + assertThat("Could not begin batch " + currentBatch, result.getValue(), is(true)); + + // Add batch data and randomize data posting order + + List dataOrder = new ArrayList(tempBatchData.length); + for (int i = 0 ; i < tempBatchData.length ; i++) { + dataOrder.add(i); + completeBatch.appendBatchData(BatchData.newBuilder() + .setData(ByteString.copyFrom(tempBatchData[i])) + .build()); + } + Collections.shuffle(dataOrder); + + // Post data + + for (int i = 0 ; i < tempBatchData.length ; i++) { + + int dataIndex = dataOrder.get(i); + + result = bulletinBoardServer.postBatchMessage(BatchMessage.newBuilder() + .setSignerId(signerIDs[0]) + .setBatchId(currentBatch) + .setSerialNum(dataIndex) + .setData(completeBatch.getBatchDataList().get(dataIndex)) + .build()); + + assertThat("Could not post batch data for batch ID " + currentBatch + " serial number " + dataIndex, + result.getValue(), is(true)); + + } + + // Close batch + + signers[0].updateContent(completeBatch); + completeBatch.setSignature(signers[0].sign()); + + result = bulletinBoardServer.closeBatchMessage(CloseBatchMessage.newBuilder() + .setBatchId(currentBatch) + .setBatchLength(tempBatchData.length) + .setSig(completeBatch.getSignature()) + .build()); + + assertThat("Could not close batch " + currentBatch, result.getValue(), is(true)); + + // Update locally stored batches + completeBatches.add(completeBatch); + + } + + public void testReadBatch() throws CommunicationException { + + for (CompleteBatch completeBatch : completeBatches) { + + List batchDataList = + bulletinBoardServer.readBatch(BatchSpecificationMessage.newBuilder() + .setSignerId(completeBatch.getBeginBatchMessage().getSignerId()) + .setBatchId(completeBatch.getBeginBatchMessage().getBatchId()) + .setStartPosition(0) + .build()); + + assertThat("Non-matching batch data for batch " + completeBatch.getBeginBatchMessage().getBatchId(), + completeBatch.getBatchDataList().equals(batchDataList), is(true)); + + + + } + + } public void close(){ signers[0].clearSigningKey(); diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java index e473931..42c11f7 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java @@ -111,6 +111,39 @@ public class MySQLBulletinBoardServerTest { System.err.println("Time of operation: " + (end - start)); } + @Test + public void testBatchPostAfterClose() { + try{ + serverTest.testBatchPostAfterClose(); + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + } + + @Test + public void testBatch() { + + final int BATCH_NUM = 20; + + try{ + for (int i = 0 ; i < BATCH_NUM ; i++) { + serverTest.testPostBatch(); + } + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + + try{ + serverTest.testReadBatch(); + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + + } + @After public void close() { System.err.println("Starting to close MySQLBulletinBoardServerTest"); diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java index 7fe3a43..0db1d3f 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java @@ -16,6 +16,7 @@ public interface BulletinBoardConstants { // Other Constants - public static final String BATCH_TAG = "Batch"; + public static final String BATCH_TAG = "@BATCH"; + public static final String BATCH_ID_TAG_PREFIX = "#"; } diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java index 6deec77..19c57e3 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java @@ -22,6 +22,7 @@ public class CompleteBatch { } public CompleteBatch(BeginBatchMessage newBeginBatchMessage) { + this(); beginBatchMessage = newBeginBatchMessage; } @@ -47,6 +48,10 @@ public class CompleteBatch { return signature; } + public void setBeginBatchMessage(BeginBatchMessage beginBatchMessage) { + this.beginBatchMessage = beginBatchMessage; + } + public void appendBatchData(BatchData newBatchData) { batchDataList.add(newBatchData); } diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java index 422e8b0..7174b42 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java @@ -3,7 +3,6 @@ package meerkat.bulletinboard; import com.google.protobuf.ByteString; import com.google.protobuf.Message; import meerkat.crypto.DigitalSignature; -import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; import meerkat.protobuf.BulletinBoardAPI.BatchData; import meerkat.protobuf.Crypto; @@ -11,10 +10,11 @@ import java.io.IOException; import java.io.InputStream; import java.security.InvalidKeyException; import java.security.KeyStore; +import java.security.KeyStoreException; import java.security.SignatureException; +import java.security.NoSuchAlgorithmException; import java.security.UnrecoverableKeyException; import java.security.cert.CertificateException; -import java.util.List; /** * Created by Arbel Deutsch Peled on 20-Dec-15. @@ -77,6 +77,12 @@ public class GenericBatchDigitalSignature implements BatchDigitalSignature{ return digitalSignature.verify(); } + @Override + public KeyStore.Builder getPKCS12KeyStoreBuilder(InputStream keyStream, char[] password) + throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException { + return digitalSignature.getPKCS12KeyStoreBuilder(keyStream, password); + } + @Override public void loadSigningCertificate(KeyStore.Builder keyStoreBuilder) throws IOException, CertificateException, UnrecoverableKeyException { digitalSignature.loadSigningCertificate(keyStoreBuilder); diff --git a/meerkat-common/src/main/java/meerkat/crypto/DigitalSignature.java b/meerkat-common/src/main/java/meerkat/crypto/DigitalSignature.java index e7b64e5..76c32a6 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/DigitalSignature.java +++ b/meerkat-common/src/main/java/meerkat/crypto/DigitalSignature.java @@ -5,11 +5,13 @@ import com.google.protobuf.Message; import java.io.IOException; import java.io.InputStream; -import java.security.InvalidKeyException; import java.security.KeyStore; -import java.security.SignatureException; -import java.security.UnrecoverableKeyException; import java.security.cert.CertificateException; +import java.security.SignatureException; +import java.security.InvalidKeyException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; import static meerkat.protobuf.Crypto.*; /** @@ -71,6 +73,20 @@ public interface DigitalSignature { */ public boolean verify(); + /** + * Load a keystore from an input stream in PKCS12 format. + * + * @param keyStream + * @param password + * @return + * @throws IOException + * @throws CertificateException + * @throws KeyStoreException + * @throws NoSuchAlgorithmException + */ + public KeyStore.Builder getPKCS12KeyStoreBuilder(InputStream keyStream, char[] password) + throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException; + /** * Loads a private signing key. The keystore must include both the public and private * key parts. diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java index 28e4600..ab8084b 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java @@ -211,6 +211,7 @@ public class ECDSASignature implements DigitalSignature { * @throws KeyStoreException * @throws NoSuchAlgorithmException */ + @Override public KeyStore.Builder getPKCS12KeyStoreBuilder(InputStream keyStream, char[] password) throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException { KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE); From d643932ef966b3aa1f01c70e351bc15029bc9ad8 Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Sun, 27 Dec 2015 12:04:37 +0200 Subject: [PATCH 06/12] Added H2 support for Batch messages --- bulletin-board-server/build.gradle | 5 ++ .../sqlserver/H2QueryProvider.java | 74 +++++++++++++++++++ .../H2BulletinBoardServerTest.java | 33 +++++++++ 3 files changed, 112 insertions(+) diff --git a/bulletin-board-server/build.gradle b/bulletin-board-server/build.gradle index 63ba47d..21604d1 100644 --- a/bulletin-board-server/build.gradle +++ b/bulletin-board-server/build.gradle @@ -84,6 +84,11 @@ task myTest(type: Test) { outputs.upToDateWhen { false } } +task h2Test(type: Test) { + include '**/*H2*Test*' + outputs.upToDateWhen { false } +} + task dbTest(type: Test) { include '**/*H2*Test*' include '**/*MySQL*Test*' diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java index d76601f..14bf9e2 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java @@ -7,6 +7,7 @@ import javax.naming.InitialContext; import javax.naming.NamingException; import javax.sql.DataSource; +import java.text.MessageFormat; import java.util.LinkedList; import java.util.List; @@ -42,6 +43,11 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider case FIND_MSG_ID: return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId"; + case FIND_TAG_ID: + return MessageFormat.format( + "SELECT TagId FROM TagTable WHERE Tag = :{0}", + QueryType.FIND_TAG_ID.getParamName(0)); + case GET_MESSAGES: return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"; @@ -55,6 +61,63 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider return "INSERT INTO TagTable(Tag) SELECT DISTINCT :Tag AS NewTag FROM UtilityTable WHERE" + " NOT EXISTS (SELECT 1 FROM TagTable AS SubTable WHERE SubTable.Tag = :Tag)"; + case GET_BATCH_MESSAGE_ENTRY: + return MessageFormat.format( + "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable" + + " INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum" + + " INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum" + + " INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId" + + " WHERE SignatureTable.SignerId = :{0}" + + " AND TagTable.Tag = :{1}", + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0), + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1)); + + case GET_BATCH_MESSAGE_DATA: + return MessageFormat.format( + "SELECT Data FROM BatchTable" + + " WHERE SignerId = :{0} AND BatchId = :{1} AND SerialNum >= :{2}" + + " ORDER BY SerialNum ASC", + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2)); + + case INSERT_BATCH_DATA: + return MessageFormat.format( + "INSERT INTO BatchTable (SignerId, BatchId, SerialNum, Data)" + + " VALUES (:{0}, :{1}, :{2}, :{3})", + QueryType.INSERT_BATCH_DATA.getParamName(0), + QueryType.INSERT_BATCH_DATA.getParamName(1), + QueryType.INSERT_BATCH_DATA.getParamName(2), + QueryType.INSERT_BATCH_DATA.getParamName(3)); + + case CHECK_BATCH_LENGTH: + return MessageFormat.format( + "SELECT COUNT(Data) AS BatchLength FROM BatchTable" + + " WHERE SignerId = :{0} AND BatchId = :{1}", + QueryType.CHECK_BATCH_LENGTH.getParamName(0), + QueryType.CHECK_BATCH_LENGTH.getParamName(1)); + + case CONNECT_BATCH_TAG: + return MessageFormat.format( + "INSERT INTO BatchTagTable (SignerId, BatchId, TagId) SELECT :{0}, :{1}, TagId FROM TagTable" + + " WHERE Tag = :{2}", + QueryType.CONNECT_BATCH_TAG.getParamName(0), + QueryType.CONNECT_BATCH_TAG.getParamName(1), + QueryType.CONNECT_BATCH_TAG.getParamName(2)); + + case GET_BATCH_TAGS: + return MessageFormat.format( + "SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.TagId" + + " WHERE SignerId = :{0} AND BatchId = :{1} ORDER BY Tag ASC", + QueryType.GET_BATCH_TAGS.getParamName(0), + QueryType.GET_BATCH_TAGS.getParamName(1)); + + case REMOVE_BATCH_TAGS: + return MessageFormat.format( + "DELETE FROM BatchTagTable WHERE SignerId = :{0} AND BatchId = :{1}", + QueryType.REMOVE_BATCH_TAGS.getParamName(0), + QueryType.REMOVE_BATCH_TAGS.getParamName(1)); + default: throw new IllegalArgumentException("Cannot serve a query of type " + queryType); } @@ -139,6 +202,14 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider list.add("CREATE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId)"); list.add("CREATE UNIQUE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId, EntryNum)"); + list.add("CREATE TABLE IF NOT EXISTS BatchTable (SignerId TINYBLOB, BatchId INT, SerialNum INT, Data BLOB," + + " UNIQUE(SignerId, BatchId, SerialNum))"); + + list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (SignerId TINYBLOB, BatchId INT, TagId INT," + + " FOREIGN KEY (TagId) REFERENCES TagTable(TagId))"); + + list.add("CREATE INDEX IF NOT EXISTS BatchIndex ON BatchTagTable(SignerId, BatchId)"); + // This is used to create a simple table with one entry. // It is used for implementing a workaround for the missing INSERT IGNORE syntax list.add("CREATE TABLE IF NOT EXISTS UtilityTable (Entry INT)"); @@ -152,6 +223,9 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider List list = new LinkedList(); list.add("DROP TABLE IF EXISTS UtilityTable"); + list.add("DROP INDEX IF EXISTS BatchIndex"); + list.add("DROP TABLE IF EXISTS BatchTagTable"); + list.add("DROP TABLE IF EXISTS BatchTable"); list.add("DROP INDEX IF EXISTS SignerIdIndex"); list.add("DROP TABLE IF EXISTS MsgTagTable"); list.add("DROP TABLE IF EXISTS SignatureTable"); diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java index ef19310..577c9be 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java @@ -107,6 +107,39 @@ public class H2BulletinBoardServerTest { System.err.println("Time of operation: " + (end - start)); } + @Test + public void testBatchPostAfterClose() { + try{ + serverTest.testBatchPostAfterClose(); + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + } + + @Test + public void testBatch() { + + final int BATCH_NUM = 20; + + try{ + for (int i = 0 ; i < BATCH_NUM ; i++) { + serverTest.testPostBatch(); + } + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + + try{ + serverTest.testReadBatch(); + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + + } + @After public void close() { System.err.println("Starting to close H2BulletinBoardServerTest"); From 141d286af26dc0663da74376f002e8b18ba6f005 Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Sun, 17 Jan 2016 10:59:05 +0200 Subject: [PATCH 07/12] Dual-layered threaded BB Client. Supports basic functionality. Does not support Batch Messages yet. --- .../bulletinboard/BulletinClientJob.java | 82 ------- .../BulletinClientJobResult.java | 29 --- .../bulletinboard/BulletinClientWorker.java | 224 ++--------------- .../bulletinboard/MultiServerWorker.java | 104 ++++++++ .../SingleServerBulletinBoardClient.java | 228 ++++++++++++++++++ .../bulletinboard/SingleServerWorker.java | 39 +++ .../ThreadedBulletinBoardClient.java | 95 ++++++-- .../callbacks/ClientFutureCallback.java | 19 -- .../GetRedundancyFutureCallback.java | 38 --- .../callbacks/PostMessageFutureCallback.java | 44 ---- .../callbacks/ReadMessagesFutureCallback.java | 35 --- .../MultiServerGetRedundancyWorker.java | 74 ++++++ .../workers/MultiServerPostBatchWorker.java | 7 + .../workers/MultiServerPostMessageWorker.java | 72 ++++++ .../MultiServerReadMessagesWorker.java | 65 +++++ .../SingleServerGetRedundancyWorker.java | 79 ++++++ .../SingleServerPostMessageWorker.java | 61 +++++ .../SingleServerReadMessagesWorker.java | 68 ++++++ .../BulletinBoardClientIntegrationTest.java | 4 +- meerkat-common/build.gradle | 2 +- .../AsyncBulletinBoardClient.java | 44 +++- 21 files changed, 927 insertions(+), 486 deletions(-) delete mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJob.java delete mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJobResult.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java delete mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java delete mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java delete mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java delete mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerGetRedundancyWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostBatchWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostMessageWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerReadMessagesWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerGetRedundancyWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerPostMessageWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerReadMessagesWorker.java diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJob.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJob.java deleted file mode 100644 index aca98d4..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJob.java +++ /dev/null @@ -1,82 +0,0 @@ -package meerkat.bulletinboard; - -import com.google.protobuf.Message; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -/** - * Created by Arbel Deutsch Peled on 09-Dec-15. - * - * This class specifies the job that is required of a Bulletin Board Client Worker - */ -public class BulletinClientJob { - - public static enum JobType{ - POST_MESSAGE, // Post a message to servers - READ_MESSAGES, // Read messages according to some given filter (any server will do) - GET_REDUNDANCY // Check the redundancy of a specific message in the databases - } - - private List serverAddresses; - - private int minServers; // The minimal number of servers the job must be successful on for the job to be completed - - private final JobType jobType; - - private final Message payload; // The information associated with the job type - - private int maxRetry; // Number of retries for this job; set to -1 for infinite retries - - public BulletinClientJob(List serverAddresses, int minServers, JobType jobType, Message payload, int maxRetry) { - this.serverAddresses = serverAddresses; - this.minServers = minServers; - this.jobType = jobType; - this.payload = payload; - this.maxRetry = maxRetry; - } - - public void updateServerAddresses(List newServerAdresses) { - this.serverAddresses = newServerAdresses; - } - - public List getServerAddresses() { - return serverAddresses; - } - - public int getMinServers() { - return minServers; - } - - public JobType getJobType() { - return jobType; - } - - public Message getPayload() { - return payload; - } - - public int getMaxRetry() { - return maxRetry; - } - - public void shuffleAddresses() { - Collections.shuffle(serverAddresses); - } - - public void decMinServers(){ - minServers--; - } - - public void decMaxRetry(){ - if (maxRetry > 0) { - maxRetry--; - } - } - - public boolean isRetry(){ - return (maxRetry != 0); - } - -} \ No newline at end of file diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJobResult.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJobResult.java deleted file mode 100644 index be0501b..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJobResult.java +++ /dev/null @@ -1,29 +0,0 @@ -package meerkat.bulletinboard; - -import com.google.protobuf.Message; - -/** - * Created by Arbel Deutsch Peled on 09-Dec-15. - * - * This class contains the end status and result of a Bulletin Board Client Job. - */ -public final class BulletinClientJobResult { - - private final BulletinClientJob job; // Stores the job the result refers to - - private final Message result; // The result of the job; valid only if success==true - - public BulletinClientJobResult(BulletinClientJob job, Message result) { - this.job = job; - this.result = result; - } - - public BulletinClientJob getJob() { - return job; - } - - public Message getResult() { - return result; - } - -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java index f03ab31..dba596b 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java @@ -1,218 +1,38 @@ package meerkat.bulletinboard; -import com.google.protobuf.Message; -import meerkat.comm.CommunicationException; -import meerkat.crypto.Digest; -import meerkat.crypto.concrete.SHA256Digest; -import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.rest.Constants; -import meerkat.rest.ProtobufMessageBodyReader; -import meerkat.rest.ProtobufMessageBodyWriter; -import static meerkat.bulletinboard.BulletinBoardConstants.*; - -import javax.ws.rs.ProcessingException; -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.Response; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.Callable; - /** * Created by Arbel Deutsch Peled on 09-Dec-15. * - * This class implements the actual communication with the Bulletin Board Servers. + * This class handles bulletin client work. * It is meant to be used in a multi-threaded environment. */ -//TODO: Maybe make this abstract and inherit from it. -public class BulletinClientWorker implements Callable { +public abstract class BulletinClientWorker { - private final BulletinClientJob job; // The requested job to be handled + protected IN payload; // Payload of the job - public BulletinClientWorker(BulletinClientJob job){ - this.job = job; + private int maxRetry; // Number of retries for this job; set to -1 for infinite retries + + public BulletinClientWorker(IN payload, int maxRetry) { + this.payload = payload; + this.maxRetry = maxRetry; } - // This resource enabled creation of a single Client per thread. - private static final ThreadLocal clientLocal = - new ThreadLocal () { - @Override protected Client initialValue() { - Client client; - client = ClientBuilder.newClient(); - client.register(ProtobufMessageBodyReader.class); - client.register(ProtobufMessageBodyWriter.class); + public IN getPayload() { + return payload; + } - return client; - } - }; - - // This resource enables creation of a single Digest per thread. - private static final ThreadLocal digestLocal = - new ThreadLocal () { - @Override protected Digest initialValue() { - Digest digest; - digest = new SHA256Digest(); //TODO: Make this generic. - - return digest; - } - }; - - /** - * This method carries out the actual communication with the servers via HTTP Post - * It accesses the servers according to the job it received and updates said job as it goes - * The method will only iterate once through the server list, removing servers from the list when they are no longer required - * In a POST_MESSAGE job: successful post to a server results in removing the server from the list - * In a GET_REDUNDANCY job: no server is removed from the list and the (absolute) number of servers in which the message was found is returned - * In a READ_MESSAGES job: successful retrieval from any server terminates the method and returns the received values; The list is not changed - * @return The original job, modified to fit the current state and the required output (if any) of the operation - * @throws IllegalArgumentException - * @throws CommunicationException - */ - public BulletinClientJobResult call() throws IllegalArgumentException, CommunicationException{ - - Client client = clientLocal.get(); - Digest digest = digestLocal.get(); - - WebTarget webTarget; - Response response; - - String requestPath; - Message msg; - - List serverAddresses = new LinkedList(job.getServerAddresses()); - - Message payload = job.getPayload(); - - BulletinBoardMessageList msgList; - - int count = 0; // Used to count number of servers which contain the required message in a GET_REDUNDANCY request. - - job.shuffleAddresses(); // This is done to randomize the order of access to servers primarily for READ operations - - // Prepare the request. - switch(job.getJobType()) { - - case POST_MESSAGE: - // Make sure the payload is a BulletinBoardMessage - if (!(payload instanceof BulletinBoardMessage)) { - throw new IllegalArgumentException("Cannot post an object that is not an instance of BulletinBoardMessage"); - } - - msg = payload; - requestPath = POST_MESSAGE_PATH; - break; - - case READ_MESSAGES: - // Make sure the payload is a MessageFilterList - if (!(payload instanceof MessageFilterList)) { - throw new IllegalArgumentException("Read failed: an instance of MessageFilterList is required as payload for a READ_MESSAGES operation"); - } - - msg = payload; - requestPath = READ_MESSAGES_PATH; - break; - - case GET_REDUNDANCY: - // Make sure the payload is a MessageId - if (!(payload instanceof MessageID)) { - throw new IllegalArgumentException("Cannot search for an object that is not an instance of MessageID"); - } - - requestPath = READ_MESSAGES_PATH; - - msg = MessageFilterList.newBuilder() - .addFilter(MessageFilter.newBuilder() - .setType(FilterType.MSG_ID) - .setId(((MessageID) payload).getID()) - .build() - ).build(); - - break; - - default: - throw new IllegalArgumentException("Unsupported job type"); - - } - - // Iterate through servers - - Iterator addressIterator = serverAddresses.iterator(); - - while (addressIterator.hasNext()) { - - // Send request to Server - String address = addressIterator.next(); - webTarget = client.target(address).path(BULLETIN_BOARD_SERVER_PATH).path(requestPath); - response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF)); - - // Retrieve answer - switch(job.getJobType()) { - - case POST_MESSAGE: - try { - - response.readEntity(BoolMsg.class); // If a BoolMsg entity is returned: the post was successful - addressIterator.remove(); // Post to this server succeeded: remove server from list - job.decMinServers(); - - } catch (ProcessingException | IllegalStateException e) {} // Post to this server failed: retry next time - finally { - response.close(); - } - break; - - case GET_REDUNDANCY: - try { - msgList = response.readEntity(BulletinBoardMessageList.class); // If a BulletinBoardMessageList is returned: the read was successful - - if (msgList.getMessageList().size() > 0){ // Message was found in the server. - count++; - } - } catch (ProcessingException | IllegalStateException e) {} // Read failed: try with next server - finally { - response.close(); - } - break; - - case READ_MESSAGES: - try { - msgList = response.readEntity(BulletinBoardMessageList.class); // If a BulletinBoardMessageList is returned: the read was successful - return new BulletinClientJobResult(job, msgList); // Return the result - } catch (ProcessingException | IllegalStateException e) {} // Read failed: try with next server - finally { - response.close(); - } - break; - - } - - } - - // Return result (if haven't done so yet) - switch(job.getJobType()) { - - case POST_MESSAGE: - // The job now contains the information required to ascertain whether enough server posts have succeeded - // It will also contain the list of servers in which the post was not successful - job.updateServerAddresses(serverAddresses); - return new BulletinClientJobResult(job, null); - - case GET_REDUNDANCY: - // Return the number of servers in which the message was found - // The job now contains the list of these servers - return new BulletinClientJobResult(job, IntMsg.newBuilder().setValue(count).build()); - - case READ_MESSAGES: - // A successful operation would have already returned an output - // Therefore: no server access was successful - throw new CommunicationException("Could not access any server"); - - default: // This is required for successful compilation - throw new IllegalArgumentException("Unsupported job type"); + public int getMaxRetry() { + return maxRetry; + } + public void decMaxRetry(){ + if (maxRetry > 0) { + maxRetry--; } } + + public boolean isRetry(){ + return (maxRetry != 0); + } + } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java new file mode 100644 index 0000000..8db836f --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java @@ -0,0 +1,104 @@ +package meerkat.bulletinboard; + +import com.google.common.util.concurrent.FutureCallback; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by Arbel Deutsch Peled on 09-Dec-15. + * + * This is a general class for handling multi-server work + * It utilizes Single Server Clients to perform the actual per-server work + */ +public abstract class MultiServerWorker extends BulletinClientWorker implements Runnable, ClientCallback{ + + private List clients; + + protected AtomicInteger minServers; // The minimal number of servers the job must be successful on for the job to be completed + + protected AtomicInteger maxFailedServers; // The maximal number of allowed server failures + + private AtomicBoolean returnedResult; + + private ClientCallback clientCallback; + + /** + * Constructor + * @param clients contains a list of Single Server clients to handle requests + * @param shuffleClients is a boolean stating whether or not it is needed to shuffle the clients + * @param minServers is the minimal amount of servers needed in order to successfully complete the job + * @param payload is the payload for the job + * @param maxRetry is the maximal per-server retry count + * @param clientCallback contains the callback methods used to report the result back to the client + */ + public MultiServerWorker(List clients, boolean shuffleClients, + int minServers, IN payload, int maxRetry, + ClientCallback clientCallback) { + + super(payload,maxRetry); + + this.clients = clients; + if (shuffleClients){ + Collections.shuffle(clients); + } + + this.minServers = new AtomicInteger(minServers); + maxFailedServers = new AtomicInteger(clients.size() - minServers); + this.clientCallback = clientCallback; + + returnedResult = new AtomicBoolean(false); + + } + + /** + * Constructor overload without client shuffling + */ + public MultiServerWorker(List clients, + int minServers, IN payload, int maxRetry, + ClientCallback clientCallback) { + + this(clients, false, minServers, payload, maxRetry, clientCallback); + + } + + /** + * Used to report a successful operation to the client + * Only reports once to the client + * @param result is the result + */ + protected void succeed(OUT result){ + if (returnedResult.compareAndSet(false, true)) { + clientCallback.handleCallback(result); + } + } + + /** + * Used to report a failed operation to the client + * Only reports once to the client + * @param t contains the error/exception that occurred + */ + protected void fail(Throwable t){ + if (returnedResult.compareAndSet(false, true)) { + clientCallback.handleFailure(t); + } + } + + /** + * Used by implementations to get a Single Server Client iterator + * @return the requested iterator + */ + protected Iterator getClientIterator() { + return clients.iterator(); + } + + protected int getClientNumber() { + return clients.size(); + } + +} \ No newline at end of file diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java new file mode 100644 index 0000000..e92f35b --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java @@ -0,0 +1,228 @@ +package meerkat.bulletinboard; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.ByteString; +import meerkat.bulletinboard.workers.SingleServerGetRedundancyWorker; +import meerkat.bulletinboard.workers.SingleServerPostMessageWorker; +import meerkat.bulletinboard.workers.SingleServerReadMessagesWorker; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Voting.BulletinBoardClientParams; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * Created by Arbel Deutsch Peled on 28-Dec-15. + * + * This class implements the asynchronous Bulletin Board Client interface + * It only handles a single Bulletin Board Server + * If the list of servers contains more than one server: the server actually used is the first one + * The class further implements a delayed access to the server after a communication error occurs + */ +public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient implements AsyncBulletinBoardClient { + + private final int MAX_RETRIES = 10; + + protected ListeningScheduledExecutorService executorService; + + private long lastServerErrorTime; + + protected final long failDelayInMilliseconds; + + /** + * Notify the client that a job has failed + * This makes new scheduled jobs be scheduled for a later time (after the given delay) + */ + protected void fail() { + + // Update last fail time + lastServerErrorTime = System.currentTimeMillis(); + + } + + /** + * This method adds a worker to the scheduled queue of the threadpool + * If the server is in an accessible state: the job is submitted for immediate handling + * If the server is not accessible: the job is scheduled for a later time + * @param worker is the worker that should be scheduled for work + * @param callback is the class containing callbacks for handling job completion/failure + */ + protected void scheduleWorker(SingleServerWorker worker, FutureCallback callback){ + + long timeSinceLastServerError = System.currentTimeMillis() - lastServerErrorTime; + + if (timeSinceLastServerError >= failDelayInMilliseconds) { + + // Schedule for immediate processing + Futures.addCallback(executorService.submit(worker), callback); + + } else { + + // Schedule for processing immediately following delay expiry + Futures.addCallback(executorService.schedule( + worker, + failDelayInMilliseconds - timeSinceLastServerError, + TimeUnit.MILLISECONDS), + callback); + + } + + } + + /** + * Inner class for handling simple operation results and retrying if needed + */ + class RetryCallback implements FutureCallback { + + private SingleServerWorker worker; + private ClientCallback clientCallback; + + public RetryCallback(SingleServerWorker worker, ClientCallback clientCallback) { + this.worker = worker; + this.clientCallback = clientCallback; + } + + @Override + public void onSuccess(T result) { + clientCallback.handleCallback(result); + } + + @Override + public void onFailure(Throwable t) { + + // Notify client about failure + fail(); + + // Check if another attempt should be made + + worker.decMaxRetry(); + + if (worker.isRetry()) { + // Perform another attempt + scheduleWorker(worker, this); + } else { + // No more retries: notify caller about failure + clientCallback.handleFailure(t); + } + + } + + } + + + public SingleServerBulletinBoardClient(int threadPoolSize, long failDelayInMilliseconds) { + + executorService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(threadPoolSize)); + + this.failDelayInMilliseconds = failDelayInMilliseconds; + + // Set server error time to a time sufficiently in the past to make new jobs go through + lastServerErrorTime = System.currentTimeMillis() - failDelayInMilliseconds; + + } + + /** + * Stores database location, initializes the web Client and + * @param clientParams contains the data needed to access the DBs + */ + @Override + public void init(BulletinBoardClientParams clientParams) { + + // Perform usual setup + super.init(clientParams); + + // Remove all but first DB address + String dbAddress = meerkatDBs.get(0); + meerkatDBs = new LinkedList(); + meerkatDBs.add(dbAddress); + + } + + @Override + public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback) { + + // Create worker with redundancy 1 and MAX_RETRIES retries + SingleServerPostMessageWorker worker = new SingleServerPostMessageWorker(meerkatDBs.get(0), msg, MAX_RETRIES); + + // Submit worker and create callback + scheduleWorker(worker, new RetryCallback(worker, callback)); + + // Calculate the correct message ID and return it + digest.reset(); + digest.update(msg.getMsg()); + return MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build(); + + } + + @Override + public MessageID postBatch(CompleteBatch completeBatch, ClientCallback callback) { + return null; + } + + @Override + public void beginBatch(byte[] signerId, int batchId, List tagList, ClientCallback callback) { + + } + + @Override + public void postBatchData(byte[] signerId, int batchId, List batchDataList, + int startPosition, ClientCallback callback) { + + } + + @Override + public void postBatchData(byte[] signerId, int batchId, List batchDataList, ClientCallback callback) { + + } + + @Override + public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback) { + + } + + @Override + public void getRedundancy(MessageID id, ClientCallback callback) { + + // Create worker with no retries + SingleServerGetRedundancyWorker worker = new SingleServerGetRedundancyWorker(meerkatDBs.get(0), id, 1); + + // Submit job and create callback + scheduleWorker(worker, new RetryCallback(worker, callback)); + + } + + @Override + public void readMessages(MessageFilterList filterList, ClientCallback> callback) { + + // Create job with no retries + SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterList, 1); + + // Submit job and create callback + scheduleWorker(worker, new RetryCallback(worker, callback)); + + } + + @Override + public void readBatch(byte[] signerId, int batchId, ClientCallback callback) { + + } + + @Override + public void subscribe(MessageFilterList filterList, MessageHandler messageHandler) { + + } + + @Override + public void close() { + + super.close(); + + executorService.shutdown(); + + } +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java new file mode 100644 index 0000000..82ca886 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java @@ -0,0 +1,39 @@ +package meerkat.bulletinboard; + +import meerkat.rest.ProtobufMessageBodyReader; +import meerkat.rest.ProtobufMessageBodyWriter; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import java.util.concurrent.Callable; + +/** + * Created by Arbel Deutsch Peled on 02-Jan-16. + */ +public abstract class SingleServerWorker extends BulletinClientWorker implements Callable{ + + // This resource enabled creation of a single Client per thread. + protected static final ThreadLocal clientLocal = + new ThreadLocal () { + @Override protected Client initialValue() { + Client client; + client = ClientBuilder.newClient(); + client.register(ProtobufMessageBodyReader.class); + client.register(ProtobufMessageBodyWriter.class); + + return client; + } + }; + + protected String serverAddress; + + public SingleServerWorker(String serverAddress, IN payload, int maxRetry) { + super(payload, maxRetry); + this.serverAddress = serverAddress; + } + + public String getServerAddress() { + return serverAddress; + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java index 71d852e..ce7a01e 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java @@ -1,15 +1,17 @@ package meerkat.bulletinboard; -import com.google.common.util.concurrent.*; import com.google.protobuf.ByteString; -import meerkat.bulletinboard.callbacks.GetRedundancyFutureCallback; -import meerkat.bulletinboard.callbacks.PostMessageFutureCallback; -import meerkat.bulletinboard.callbacks.ReadMessagesFutureCallback; + +import meerkat.bulletinboard.workers.MultiServerGetRedundancyWorker; +import meerkat.bulletinboard.workers.MultiServerPostMessageWorker; +import meerkat.bulletinboard.workers.MultiServerReadMessagesWorker; import meerkat.comm.CommunicationException; import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.protobuf.Voting.*; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -22,10 +24,19 @@ import java.util.concurrent.TimeUnit; */ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient implements AsyncBulletinBoardClient { - private final static int THREAD_NUM = 10; - ListeningExecutorService listeningExecutor; + // Executor service for handling jobs + private final static int JOBS_THREAD_NUM = 5; + ExecutorService executorService; + // Per-server clients + List clients; + + private final static int POST_MESSAGE_RETRY_NUM = 3; private final static int READ_MESSAGES_RETRY_NUM = 1; + private final static int GET_REDUNDANCY_RETRY_NUM = 1; + + private static final int SERVER_THREADPOOL_SIZE = 5; + private static final long FAIL_DELAY = 5000; private int minAbsoluteRedundancy; @@ -42,7 +53,16 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * clientParams.getBulletinBoardAddressCount()); - listeningExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(THREAD_NUM)); + executorService = Executors.newFixedThreadPool(JOBS_THREAD_NUM); + + clients = new ArrayList(clientParams.getBulletinBoardAddressCount()); + for (String address : clientParams.getBulletinBoardAddressList()){ + SingleServerBulletinBoardClient client = new SingleServerBulletinBoardClient(SERVER_THREADPOOL_SIZE, FAIL_DELAY); + client.init(BulletinBoardClientParams.newBuilder() + .addBulletinBoardAddress(address) + .build()); + clients.add(client); + } } @@ -54,28 +74,52 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple * @throws CommunicationException */ @Override - public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback){ + public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback){ // Create job - BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.POST_MESSAGE, msg, -1); + MultiServerPostMessageWorker worker = + new MultiServerPostMessageWorker(clients, minAbsoluteRedundancy, msg, POST_MESSAGE_RETRY_NUM, callback); - // Submit job and create callback - Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new PostMessageFutureCallback(listeningExecutor, callback)); + // Submit job + executorService.submit(worker); // Calculate the correct message ID and return it digest.reset(); digest.update(msg.getMsg()); return MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build(); + } @Override - public MessageID postBatch(byte[] signerId, int batchId, List batchDataList, int startPosition, ClientCallback callback) { - return null; //TODO: Implement + public MessageID postBatch(CompleteBatch completeBatch, ClientCallback callback) { + + return null; // TODO: write this + } @Override - public MessageID postBatch(byte[] signerId, int batchId, List batchDataList, ClientCallback callback) { - return null; //TODO: Implement + public void beginBatch(byte[] signerId, int batchId, List tagList, ClientCallback callback) { + // TODO: write this + } + + @Override + public void postBatchData(byte[] signerId, int batchId, List batchDataList, + int startPosition, ClientCallback callback) { + + // TODO: write this + + } + + @Override + public void postBatchData(byte[] signerId, int batchId, List batchDataList, ClientCallback callback) { + + postBatchData(signerId, batchId, batchDataList, 0, callback); // Write batch from beginning + + } + + @Override + public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback) { + // TODO: write this } /** @@ -88,10 +132,11 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple public void getRedundancy(MessageID id, ClientCallback callback) { // Create job - BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.GET_REDUNDANCY, id, 1); + MultiServerGetRedundancyWorker worker = + new MultiServerGetRedundancyWorker(clients, minAbsoluteRedundancy, id, GET_REDUNDANCY_RETRY_NUM, callback); - // Submit job and create callback - Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new GetRedundancyFutureCallback(listeningExecutor, callback)); + // Submit job + executorService.submit(worker); } @@ -104,11 +149,11 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple public void readMessages(MessageFilterList filterList, ClientCallback> callback) { // Create job - BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.READ_MESSAGES, - filterList, READ_MESSAGES_RETRY_NUM); + MultiServerReadMessagesWorker worker = + new MultiServerReadMessagesWorker(clients, minAbsoluteRedundancy, filterList, READ_MESSAGES_RETRY_NUM, callback); - // Submit job and create callback - Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new ReadMessagesFutureCallback(listeningExecutor, callback)); + // Submit job + executorService.submit(worker); } @@ -127,9 +172,9 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple super.close(); try { - listeningExecutor.shutdown(); - while (! listeningExecutor.isShutdown()) { - listeningExecutor.awaitTermination(10, TimeUnit.SECONDS); + executorService.shutdown(); + while (! executorService.isShutdown()) { + executorService.awaitTermination(10, TimeUnit.SECONDS); } } catch (InterruptedException e) { System.err.println(e.getCause() + " " + e.getMessage()); diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java deleted file mode 100644 index 54cc63a..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java +++ /dev/null @@ -1,19 +0,0 @@ -package meerkat.bulletinboard.callbacks; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.BulletinClientJobResult; - -/** - * This is a future callback used to listen to workers and run on job finish - * Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error - */ -public abstract class ClientFutureCallback implements FutureCallback { - - protected ListeningExecutorService listeningExecutor; - - ClientFutureCallback(ListeningExecutorService listeningExecutor) { - this.listeningExecutor = listeningExecutor; - } - -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java deleted file mode 100644 index 719428f..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java +++ /dev/null @@ -1,38 +0,0 @@ -package meerkat.bulletinboard.callbacks; - -import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.AsyncBulletinBoardClient.*; -import meerkat.bulletinboard.BulletinClientJobResult; -import meerkat.protobuf.BulletinBoardAPI.*; - -import java.util.List; - -/** - * This is a future callback used to listen to workers and run on job finish - * Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error - */ -public class GetRedundancyFutureCallback extends ClientFutureCallback { - - private ClientCallback callback; - - public GetRedundancyFutureCallback(ListeningExecutorService listeningExecutor, - ClientCallback callback) { - super(listeningExecutor); - this.callback = callback; - } - - @Override - public void onSuccess(BulletinClientJobResult result) { - - int absoluteRedundancy = ((IntMsg) result.getResult()).getValue(); - int totalServers = result.getJob().getServerAddresses().size(); - - callback.handleCallback( ((float) absoluteRedundancy) / ((float) totalServers) ); - - } - - @Override - public void onFailure(Throwable t) { - callback.handleFailure(t); - } -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java deleted file mode 100644 index abd4247..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java +++ /dev/null @@ -1,44 +0,0 @@ -package meerkat.bulletinboard.callbacks; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.AsyncBulletinBoardClient.*; -import meerkat.bulletinboard.BulletinClientJob; -import meerkat.bulletinboard.BulletinClientJobResult; -import meerkat.bulletinboard.BulletinClientWorker; - - -/** - * This is a future callback used to listen to workers and run on job finish - * Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error - */ -public class PostMessageFutureCallback extends ClientFutureCallback { - - private ClientCallback callback; - - public PostMessageFutureCallback(ListeningExecutorService listeningExecutor, - ClientCallback callback) { - super(listeningExecutor); - this.callback = callback; - } - - @Override - public void onSuccess(BulletinClientJobResult result) { - - BulletinClientJob job = result.getJob(); - - job.decMaxRetry(); - - // If redundancy is below threshold: retry - if (job.getMinServers() > 0 && job.isRetry()) { - Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), this); - } - - callback.handleCallback(null); - } - - @Override - public void onFailure(Throwable t) { - callback.handleFailure(t); - } -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java deleted file mode 100644 index 808b7a6..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java +++ /dev/null @@ -1,35 +0,0 @@ -package meerkat.bulletinboard.callbacks; - -import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.AsyncBulletinBoardClient.*; -import meerkat.bulletinboard.BulletinClientJobResult; -import meerkat.protobuf.BulletinBoardAPI; - -import java.util.List; - -/** - * This is a future callback used to listen to workers and run on job finish - * Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error - */ -public class ReadMessagesFutureCallback extends ClientFutureCallback { - - private ClientCallback> callback; - - public ReadMessagesFutureCallback(ListeningExecutorService listeningExecutor, - ClientCallback> callback) { - super(listeningExecutor); - this.callback = callback; - } - - @Override - public void onSuccess(BulletinClientJobResult result) { - - callback.handleCallback(((BulletinBoardAPI.BulletinBoardMessageList) result.getResult()).getMessageList()); - - } - - @Override - public void onFailure(Throwable t) { - callback.handleFailure(t); - } -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerGetRedundancyWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerGetRedundancyWorker.java new file mode 100644 index 0000000..4d5e7f2 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerGetRedundancyWorker.java @@ -0,0 +1,74 @@ +package meerkat.bulletinboard.workers; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.MultiServerWorker; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerGetRedundancyWorker extends MultiServerWorker { + + private AtomicInteger serversContainingMessage; + private AtomicInteger totalContactedServers; + + public MultiServerGetRedundancyWorker(List clients, + int minServers, MessageID payload, int maxRetry, + ClientCallback clientCallback) { + + super(clients, minServers, payload, maxRetry, clientCallback); // Shuffle clients on creation to balance load + + serversContainingMessage = new AtomicInteger(0); + totalContactedServers = new AtomicInteger(0); + + } + + /** + * This method carries out the actual communication with the servers via HTTP Post + * It accesses the servers in a random order until one answers it + * Successful retrieval from any server terminates the method and returns the received values; The list is not changed + * @return The original job and the list of messages found in the first server that answered the query + * @throws CommunicationException + */ + public void run(){ + + Iterator clientIterator = getClientIterator(); + + // Iterate through clients + + while (clientIterator.hasNext()) { + + SingleServerBulletinBoardClient client = clientIterator.next(); + + // Send request to client + client.getRedundancy(payload,this); + + } + + } + + @Override + public void handleCallback(Float result) { + + if (result > 0.5) { + serversContainingMessage.incrementAndGet(); + } + + if (totalContactedServers.incrementAndGet() >= getClientNumber()){ + succeed(new Float(((float) serversContainingMessage.get()) / ((float) getClientNumber()) )); + } + + } + + @Override + public void handleFailure(Throwable t) { + handleCallback(new Float(0.0)); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostBatchWorker.java new file mode 100644 index 0000000..cfc34e7 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostBatchWorker.java @@ -0,0 +1,7 @@ +package meerkat.bulletinboard.workers; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerPostBatchWorker { +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostMessageWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostMessageWorker.java new file mode 100644 index 0000000..dcdc82a --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostMessageWorker.java @@ -0,0 +1,72 @@ +package meerkat.bulletinboard.workers; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.MultiServerWorker; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; + +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; +import java.util.Iterator; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerPostMessageWorker extends MultiServerWorker { + + public MultiServerPostMessageWorker(List clients, + int minServers, BulletinBoardMessage payload, int maxRetry, + ClientCallback clientCallback) { + + super(clients, minServers, payload, maxRetry, clientCallback); + + } + + /** + * This method carries out the actual communication with the servers via HTTP Post + * It accesses the servers one by one and tries to post the payload to each in turn + * The method will only iterate once through the server list + * Successful post to a server results in removing the server from the list + * @return The original job, but with a modified server list + * @throws CommunicationException + */ + public void run() { + + WebTarget webTarget; + Response response; + + int count = 0; // Used to count number of servers which contain the required message in a GET_REDUNDANCY request. + + // Iterate through servers + + Iterator clientIterator = getClientIterator(); + + while (clientIterator.hasNext()) { + + // Send request to Server + SingleServerBulletinBoardClient client = clientIterator.next(); + + client.postMessage(payload, this); + + } + + } + + @Override + public void handleCallback(Boolean result) { + if (result){ + if (minServers.decrementAndGet() <= 0){ + succeed(Boolean.TRUE); + } + } + } + + @Override + public void handleFailure(Throwable t) { + if (maxFailedServers.decrementAndGet() < 0){ + fail(t); + } + } +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerReadMessagesWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerReadMessagesWorker.java new file mode 100644 index 0000000..bf19526 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerReadMessagesWorker.java @@ -0,0 +1,65 @@ +package meerkat.bulletinboard.workers; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.MultiServerWorker; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.Iterator; +import java.util.List; + + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerReadMessagesWorker extends MultiServerWorker>{ + + private Iterator clientIterator; + + public MultiServerReadMessagesWorker(List clients, + int minServers, MessageFilterList payload, int maxRetry, + ClientCallback> clientCallback) { + + super(clients, true, minServers, payload, maxRetry, clientCallback); // Shuffle clients on creation to balance load + + clientIterator = getClientIterator(); + + } + + /** + * This method carries out the actual communication with the servers via HTTP Post + * It accesses the servers in a random order until one answers it + * Successful retrieval from any server terminates the method and returns the received values; The list is not changed + * @return The original job and the list of messages found in the first server that answered the query + * @throws CommunicationException + */ + public void run(){ + + // Iterate through servers + + if (clientIterator.hasNext()) { + + // Send request to Server + SingleServerBulletinBoardClient client = clientIterator.next(); + + // Retrieve answer + client.readMessages(payload,this); + + } else { + fail(new CommunicationException("Could not contact any server")); + } + + } + + @Override + public void handleCallback(List msg) { + succeed(msg); + } + + @Override + public void handleFailure(Throwable t) { + run(); // Retry with next server + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerGetRedundancyWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerGetRedundancyWorker.java new file mode 100644 index 0000000..a02da03 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerGetRedundancyWorker.java @@ -0,0 +1,79 @@ +package meerkat.bulletinboard.workers; + +import meerkat.bulletinboard.SingleServerWorker; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.rest.Constants; + +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; + +import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; +import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class SingleServerGetRedundancyWorker extends SingleServerWorker { + + public SingleServerGetRedundancyWorker(String serverAddress, MessageID payload, int maxRetry) { + super(serverAddress, payload, maxRetry); + } + + /** + * This method carries out the actual communication with the server via HTTP Post + * It queries the server for a message with the given ID + * @return TRUE if the message exists in the server and FALSE otherwise + * @throws CommunicationException if the server does not return a valid answer + */ + public Float call() throws CommunicationException{ + + Client client = clientLocal.get(); + + WebTarget webTarget; + Response response; + + MessageFilterList msgFilterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.MSG_ID) + .setId(payload.getID()) + .build() + ).build(); + + // Send request to Server + + webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); + response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msgFilterList, Constants.MEDIATYPE_PROTOBUF)); + + // Retrieve answer + + try { + + // If a BulletinBoardMessageList is returned: the read was successful + BulletinBoardMessageList msgList = response.readEntity(BulletinBoardMessageList.class); + + if (msgList.getMessageList().size() > 0){ + // Message exists in the server + return new Float(1.0); + } + else { + // Message does not exist in the server + return new Float(0.0); + } + + } catch (ProcessingException | IllegalStateException e) { + + // Read failed + throw new CommunicationException("Server access failed"); + + } + finally { + response.close(); + } + + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerPostMessageWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerPostMessageWorker.java new file mode 100644 index 0000000..ee9fd3f --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerPostMessageWorker.java @@ -0,0 +1,61 @@ +package meerkat.bulletinboard.workers; + +import meerkat.bulletinboard.SingleServerWorker; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.BoolMsg; +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; +import meerkat.rest.Constants; + +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; + +import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; +import static meerkat.bulletinboard.BulletinBoardConstants.POST_MESSAGE_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + * Tries to contact server once and perform a post operation + */ +public class SingleServerPostMessageWorker extends SingleServerWorker { + + public SingleServerPostMessageWorker(String serverAddress, BulletinBoardMessage payload, int maxRetry) { + super(serverAddress, payload, maxRetry); + } + + /** + * This method carries out the actual communication with the server via HTTP Post + * It accesses the server and tries to post the payload to it + * Successful post to a server results + * @return TRUE if the operation is successful + * @throws CommunicationException if the operation is unseccessful + */ + public Boolean call() throws CommunicationException{ + + Client client = clientLocal.get(); + + WebTarget webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(POST_MESSAGE_PATH);; + Response response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post( + Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF)); + + try { + + // If a BoolMsg entity is returned: the post was successful + response.readEntity(BoolMsg.class); + return Boolean.TRUE; + + } catch (ProcessingException | IllegalStateException e) { + + // Post to this server failed + throw new CommunicationException("Could not contact the server"); + + } + finally { + response.close(); + } + + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerReadMessagesWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerReadMessagesWorker.java new file mode 100644 index 0000000..f8975cb --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerReadMessagesWorker.java @@ -0,0 +1,68 @@ +package meerkat.bulletinboard.workers; + +import meerkat.bulletinboard.SingleServerWorker; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessageList; +import meerkat.protobuf.BulletinBoardAPI.MessageFilterList; +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; +import meerkat.rest.Constants; + +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; + +import java.util.List; + +import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; +import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class SingleServerReadMessagesWorker extends SingleServerWorker> { + + public SingleServerReadMessagesWorker(String serverAddress, MessageFilterList payload, int maxRetry) { + super(serverAddress, payload, maxRetry); + } + + /** + * This method carries out the actual communication with the server via HTTP Post + * Upon successful retrieval from the server the method returns the received values + * @return The list of messages returned by the server + * @throws CommunicationException if the server's response is invalid + */ + public List call() throws CommunicationException{ + + Client client = clientLocal.get(); + + WebTarget webTarget; + Response response; + + // Send request to Server + webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); + response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post( + Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF)); + + // Retrieve answer + + try { + + // If a BulletinBoardMessageList is returned: the read was successful + return response.readEntity(BulletinBoardMessageList.class).getMessageList(); + + } catch (ProcessingException | IllegalStateException e) { + + // Read failed + throw new CommunicationException("Could not contact the server"); + + } + finally { + response.close(); + } + + + } + +} diff --git a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java b/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java index d7ae69c..7bdbe08 100644 --- a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java +++ b/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java @@ -32,10 +32,10 @@ public class BulletinBoardClientIntegrationTest { jobSemaphore.release(); } - private class PostCallback implements ClientCallback{ + private class PostCallback implements ClientCallback{ @Override - public void handleCallback(Object msg) { + public void handleCallback(Boolean msg) { System.err.println("Post operation completed"); jobSemaphore.release(); } diff --git a/meerkat-common/build.gradle b/meerkat-common/build.gradle index c510e0a..3783531 100644 --- a/meerkat-common/build.gradle +++ b/meerkat-common/build.gradle @@ -46,7 +46,7 @@ dependencies { compile 'com.google.protobuf:protobuf-java:3.+' // ListeningExecutor - compile 'com.google.guava:guava:11.0.+' + compile 'com.google.guava:guava:15.0' // Crypto compile 'org.factcenter.qilin:qilin:1.2+' diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java index 1663f7a..00bd2ac 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java @@ -1,6 +1,7 @@ package meerkat.bulletinboard; import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Crypto.Signature; import java.util.List; @@ -24,23 +25,48 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param callback is a class containing methods to handle the result of the operation * @return a unique message ID for the message, that can be later used to retrieve the batch */ - public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback); + public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback); /** - * This method allows for sending large messages as a batch to the bulletin board + * Perform an end-to-end post of a signed batch message + * @param completeBatch contains all the data of the batch including the meta-data and the signature + * @param callback is a class containing methods to handle the result of the operation + * @return a unique identifier for the batch message + */ + public MessageID postBatch(CompleteBatch completeBatch, ClientCallback callback); + + /** + * This message informs the server about the existence of a new batch message and supplies it with the tags associated with it * @param signerId is the canonical form for the ID of the sender of this batch * @param batchId is a unique (per signer) ID for this batch - * @param batchDataList is the (canonically ordered) list of data comprising the batch message - * @param startPosition is the location (in the batch) of the first entry in batchDataList (optionally used to continue interrupted post operations) - * @param callback is a callback function class for handling results of the operation - * @return a unique message ID for the entire message, that can be later used to retrieve the batch + * @param tagList is a list of tags that belong to the batch message */ - public MessageID postBatch(byte[] signerId, int batchId, List batchDataList, int startPosition, ClientCallback callback); + public void beginBatch(byte[] signerId, int batchId, List tagList, ClientCallback callback); /** - * Overloading of the postBatch method in which startPosition is set to the default value 0 + * This method posts batch data into an (assumed to be open) batch + * It does not close the batch + * @param signerId is the canonical form for the ID of the sender of this batch + * @param batchId is a unique (per signer) ID for this batch + * @param batchDataList is the (canonically ordered) list of data comprising the entire batch message (not just the portion to be written) + * @param startPosition is the location (in the batch) of the first entry in batchDataList + * (optionally used to continue interrupted post operations) + * @param callback is a callback function class for handling results of the operation */ - public MessageID postBatch(byte[] signerId, int batchId, List batchDataList, ClientCallback callback); + public void postBatchData(byte[] signerId, int batchId, List batchDataList, + int startPosition, ClientCallback callback); + + /** + * Overloading of the postBatchData method in which startPosition is set to the default value 0 + */ + public void postBatchData(byte[] signerId, int batchId, List batchDataList, ClientCallback callback); + + /** + * Attempts to close a batch message + * @param closeBatchMessage contains the data required to close the batch + * @param callback is a callback function class for handling results of the operation + */ + public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback); /** * Check how "safe" a given message is in an asynchronous manner From 3fed32f9e62d83cc1a67f7a463032a86b46eb30e Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Sun, 17 Jan 2016 19:57:45 +0200 Subject: [PATCH 08/12] First (untested) version of BB Client with full batch support --- .../bulletinboard/BatchDataContainer.java | 25 +++ .../SingleServerBulletinBoardClient.java | 145 ++++++++++++++++-- .../ThreadedBulletinBoardClient.java | 87 +++++++++-- .../workers/MultiServerPostBatchWorker.java | 7 - .../MultiServerBeginBatchWorker.java | 28 ++++ .../MultiServerCloseBatchWorker.java | 28 ++++ .../MultiServerGenericPostWorker.java} | 13 +- .../MultiServerGenericReadWorker.java} | 19 +-- .../MultiServerGetRedundancyWorker.java | 2 +- .../MultiServerPostBatchDataWorker.java | 28 ++++ .../MultiServerPostBatchWorker.java | 28 ++++ .../MultiServerPostMessageWorker.java | 28 ++++ .../MultiServerReadBatchWorker.java | 30 ++++ .../MultiServerReadMessagesWorker.java | 29 ++++ .../SingleServerBeginBatchWorker.java | 17 ++ .../SingleServerCloseBatchWorker.java | 17 ++ .../SingleServerGenericPostWorker.java} | 15 +- .../SingleServerGetRedundancyWorker.java | 2 +- .../SingleServerPostBatchWorker.java | 17 ++ .../SingleServerPostMessageWorker.java | 17 ++ .../SingleServerReadBatchWorker.java | 119 ++++++++++++++ .../SingleServerReadMessagesWorker.java | 2 +- .../webapp/BulletinBoardWebApp.java | 4 + .../AsyncBulletinBoardClient.java | 34 ++-- .../bulletinboard/BulletinBoardConstants.java | 3 +- .../bulletinboard/GenericBatchDigest.java | 7 +- .../src/main/java/meerkat/crypto/Digest.java | 7 + .../meerkat/crypto/concrete/SHA256Digest.java | 6 + 28 files changed, 692 insertions(+), 72 deletions(-) create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/BatchDataContainer.java delete mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostBatchWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java rename bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/{MultiServerPostMessageWorker.java => multiserver/MultiServerGenericPostWorker.java} (82%) rename bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/{MultiServerReadMessagesWorker.java => multiserver/MultiServerGenericReadWorker.java} (68%) rename bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/{ => multiserver}/MultiServerGetRedundancyWorker.java (97%) create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchDataWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerBeginBatchWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerCloseBatchWorker.java rename bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/{SingleServerPostMessageWorker.java => singleserver/SingleServerGenericPostWorker.java} (76%) rename bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/{ => singleserver}/SingleServerGetRedundancyWorker.java (98%) create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostBatchWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostMessageWorker.java create mode 100644 bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java rename bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/{ => singleserver}/SingleServerReadMessagesWorker.java (97%) diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BatchDataContainer.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BatchDataContainer.java new file mode 100644 index 0000000..53f4870 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BatchDataContainer.java @@ -0,0 +1,25 @@ +package meerkat.bulletinboard; + +import com.google.protobuf.ByteString; +import meerkat.protobuf.BulletinBoardAPI.BatchData; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 17-Jan-16. + * Used to store the complete data required for sending a batch data list inside a single object + */ +public class BatchDataContainer { + + public final byte[] signerId; + public final int batchId; + public final List batchDataList; + public final int startPosition; + + public BatchDataContainer(byte[] signerId, int batchId, List batchDataList, int startPosition) { + this.signerId = signerId; + this.batchId = batchId; + this.batchDataList = batchDataList; + this.startPosition = startPosition; + } +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java index e92f35b..9c211fd 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java @@ -5,9 +5,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; -import meerkat.bulletinboard.workers.SingleServerGetRedundancyWorker; -import meerkat.bulletinboard.workers.SingleServerPostMessageWorker; -import meerkat.bulletinboard.workers.SingleServerReadMessagesWorker; +import meerkat.bulletinboard.workers.singleserver.*; import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.protobuf.Voting.BulletinBoardClientParams; @@ -30,6 +28,8 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i protected ListeningScheduledExecutorService executorService; + protected BatchDigest batchDigest; + private long lastServerErrorTime; protected final long failDelayInMilliseconds; @@ -136,6 +136,9 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i // Perform usual setup super.init(clientParams); + // Wrap the Digest into a BatchDigest + batchDigest = new GenericBatchDigest(digest); + // Remove all but first DB address String dbAddress = meerkatDBs.get(0); meerkatDBs = new LinkedList(); @@ -153,19 +156,124 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i scheduleWorker(worker, new RetryCallback(worker, callback)); // Calculate the correct message ID and return it - digest.reset(); - digest.update(msg.getMsg()); - return MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build(); + batchDigest.reset(); + batchDigest.update(msg.getMsg()); + return batchDigest.digestAsMessageID(); } + private class PostBatchDataCallback implements ClientCallback { + + private CompleteBatch completeBatch; + ClientCallback callback; + + public PostBatchDataCallback(CompleteBatch completeBatch, ClientCallback callback) { + this.completeBatch = completeBatch; + this.callback = callback; + } + + @Override + public void handleCallback(Boolean msg) { + closeBatch( + CloseBatchMessage.newBuilder() + .setBatchId(completeBatch.getBeginBatchMessage().getBatchId()) + .setSig(completeBatch.getSignature()) + .setBatchLength(completeBatch.getBatchDataList().size()) + .build(), + callback + ); + } + + @Override + public void handleFailure(Throwable t) { + callback.handleFailure(t); + } + + } + + private class BeginBatchCallback implements ClientCallback { + + private CompleteBatch completeBatch; + ClientCallback callback; + + public BeginBatchCallback(CompleteBatch completeBatch, ClientCallback callback) { + this.completeBatch = completeBatch; + this.callback = callback; + } + + @Override + public void handleCallback(Boolean msg) { + + postBatchData( + completeBatch.getBeginBatchMessage().getSignerId(), + completeBatch.getBeginBatchMessage().getBatchId(), + completeBatch.getBatchDataList(), + 0, + new PostBatchDataCallback(completeBatch,callback)); + } + + @Override + public void handleFailure(Throwable t) { + callback.handleFailure(t); + } + } + @Override public MessageID postBatch(CompleteBatch completeBatch, ClientCallback callback) { - return null; + + beginBatch( + completeBatch.getBeginBatchMessage(), + new BeginBatchCallback(completeBatch, callback) + ); + + batchDigest.update(completeBatch); + + return batchDigest.digestAsMessageID(); + } @Override - public void beginBatch(byte[] signerId, int batchId, List tagList, ClientCallback callback) { + public void beginBatch(BeginBatchMessage beginBatchMessage, ClientCallback callback) { + + // Create worker with redundancy 1 and MAX_RETRIES retries + SingleServerBeginBatchWorker worker = + new SingleServerBeginBatchWorker(meerkatDBs.get(0), beginBatchMessage, MAX_RETRIES); + + // Submit worker and create callback + scheduleWorker(worker, new RetryCallback(worker, callback)); + + } + + @Override + public void postBatchData(ByteString signerId, int batchId, List batchDataList, + int startPosition, ClientCallback callback) { + + BatchMessage.Builder builder = BatchMessage.newBuilder() + .setSignerId(signerId) + .setBatchId(batchId); + + // Iterate through data list + + for (BatchData data : batchDataList) { + builder.setSerialNum(startPosition).setData(data); + + // Create worker with redundancy 1 and MAX_RETRIES retries + SingleServerPostBatchWorker worker = + new SingleServerPostBatchWorker(meerkatDBs.get(0), builder.build(), MAX_RETRIES); + + // Create worker with redundancy 1 and MAX_RETRIES retries + scheduleWorker(worker, new RetryCallback(worker, callback)); + + // Increment position in batch + startPosition++; + } + + } + + @Override + public void postBatchData(ByteString signerId, int batchId, List batchDataList, ClientCallback callback) { + + postBatchData(signerId, batchId, batchDataList, 0, callback); } @@ -173,15 +281,26 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i public void postBatchData(byte[] signerId, int batchId, List batchDataList, int startPosition, ClientCallback callback) { + postBatchData(ByteString.copyFrom(signerId), batchId, batchDataList, startPosition, callback); + } @Override public void postBatchData(byte[] signerId, int batchId, List batchDataList, ClientCallback callback) { + postBatchData(signerId, batchId, batchDataList, 0, callback); + } @Override - public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback) { + public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback) { + + // Create worker with redundancy 1 and MAX_RETRIES retries + SingleServerCloseBatchWorker worker = + new SingleServerCloseBatchWorker(meerkatDBs.get(0), closeBatchMessage, MAX_RETRIES); + + // Submit worker and create callback + scheduleWorker(worker, new RetryCallback(worker, callback)); } @@ -208,7 +327,13 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } @Override - public void readBatch(byte[] signerId, int batchId, ClientCallback callback) { + public void readBatch(BatchSpecificationMessage batchSpecificationMessage, ClientCallback callback) { + + // Create job with no retries + SingleServerReadBatchWorker worker = new SingleServerReadBatchWorker(meerkatDBs.get(0), batchSpecificationMessage, 1); + + // Submit job and create callback + scheduleWorker(worker, new RetryCallback(worker, callback)); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java index ce7a01e..78d5dba 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java @@ -2,9 +2,7 @@ package meerkat.bulletinboard; import com.google.protobuf.ByteString; -import meerkat.bulletinboard.workers.MultiServerGetRedundancyWorker; -import meerkat.bulletinboard.workers.MultiServerPostMessageWorker; -import meerkat.bulletinboard.workers.MultiServerReadMessagesWorker; +import meerkat.bulletinboard.workers.multiserver.*; import meerkat.comm.CommunicationException; import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.protobuf.Voting.*; @@ -31,6 +29,8 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple // Per-server clients List clients; + BatchDigest batchDigest; + private final static int POST_MESSAGE_RETRY_NUM = 3; private final static int READ_MESSAGES_RETRY_NUM = 1; private final static int GET_REDUNDANCY_RETRY_NUM = 1; @@ -51,6 +51,8 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple super.init(clientParams); + batchDigest = new GenericBatchDigest(digest); + minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * clientParams.getBulletinBoardAddressCount()); executorService = Executors.newFixedThreadPool(JOBS_THREAD_NUM); @@ -84,42 +86,88 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple executorService.submit(worker); // Calculate the correct message ID and return it - digest.reset(); - digest.update(msg.getMsg()); - return MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build(); + batchDigest.reset(); + batchDigest.update(msg.getMsg()); + return batchDigest.digestAsMessageID(); } @Override public MessageID postBatch(CompleteBatch completeBatch, ClientCallback callback) { - return null; // TODO: write this + // Create job + MultiServerPostBatchWorker worker = + new MultiServerPostBatchWorker(clients, minAbsoluteRedundancy, completeBatch, POST_MESSAGE_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); + + // Calculate the correct message ID and return it + batchDigest.reset(); + batchDigest.update(completeBatch); + return batchDigest.digestAsMessageID(); } @Override - public void beginBatch(byte[] signerId, int batchId, List tagList, ClientCallback callback) { - // TODO: write this + public void beginBatch(BeginBatchMessage beginBatchMessage, ClientCallback callback) { + + // Create job + MultiServerBeginBatchWorker worker = + new MultiServerBeginBatchWorker(clients, minAbsoluteRedundancy, beginBatchMessage, POST_MESSAGE_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); + } @Override public void postBatchData(byte[] signerId, int batchId, List batchDataList, - int startPosition, ClientCallback callback) { + int startPosition, ClientCallback callback) { - // TODO: write this + BatchDataContainer batchDataContainer = new BatchDataContainer(signerId, batchId, batchDataList, startPosition); + + // Create job + MultiServerPostBatchDataWorker worker = + new MultiServerPostBatchDataWorker(clients, minAbsoluteRedundancy, batchDataContainer, POST_MESSAGE_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); } @Override public void postBatchData(byte[] signerId, int batchId, List batchDataList, ClientCallback callback) { - postBatchData(signerId, batchId, batchDataList, 0, callback); // Write batch from beginning + postBatchData(signerId, batchId, batchDataList, 0, callback); } @Override - public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback) { - // TODO: write this + public void postBatchData(ByteString signerId, int batchId, List batchDataList, + int startPosition, ClientCallback callback) { + + postBatchData(signerId.toByteArray(), batchId, batchDataList, startPosition, callback); + + } + + @Override + public void postBatchData(ByteString signerId, int batchId, List batchDataList, ClientCallback callback) { + + postBatchData(signerId, batchId, batchDataList, 0, callback); + + } + + @Override + public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback) { + + // Create job + MultiServerCloseBatchWorker worker = + new MultiServerCloseBatchWorker(clients, minAbsoluteRedundancy, closeBatchMessage, POST_MESSAGE_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); + } /** @@ -158,8 +206,15 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple } @Override - public void readBatch(byte[] signerId, int batchId, ClientCallback callback) { - // TODO: Implement + public void readBatch(BatchSpecificationMessage batchSpecificationMessage, ClientCallback callback) { + + // Create job + MultiServerReadBatchWorker worker = + new MultiServerReadBatchWorker(clients, minAbsoluteRedundancy, batchSpecificationMessage, READ_MESSAGES_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); + } @Override diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostBatchWorker.java deleted file mode 100644 index cfc34e7..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostBatchWorker.java +++ /dev/null @@ -1,7 +0,0 @@ -package meerkat.bulletinboard.workers; - -/** - * Created by Arbel Deutsch Peled on 27-Dec-15. - */ -public class MultiServerPostBatchWorker { -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java new file mode 100644 index 0000000..dc496d7 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerBeginBatchWorker extends MultiServerGenericPostWorker { + + public MultiServerBeginBatchWorker(List clients, + int minServers, BeginBatchMessage payload, int maxRetry, + ClientCallback clientCallback) { + + super(clients, minServers, payload, maxRetry, clientCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, BeginBatchMessage payload) { + client.beginBatch(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java new file mode 100644 index 0000000..56b09c5 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.CloseBatchMessage; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerCloseBatchWorker extends MultiServerGenericPostWorker { + + public MultiServerCloseBatchWorker(List clients, + int minServers, CloseBatchMessage payload, int maxRetry, + ClientCallback clientCallback) { + + super(clients, minServers, payload, maxRetry, clientCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, CloseBatchMessage payload) { + client.closeBatch(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostMessageWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java similarity index 82% rename from bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostMessageWorker.java rename to bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java index dcdc82a..8b62d4e 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerPostMessageWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java @@ -1,10 +1,9 @@ -package meerkat.bulletinboard.workers; +package meerkat.bulletinboard.workers.multiserver; import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; import meerkat.bulletinboard.MultiServerWorker; import meerkat.bulletinboard.SingleServerBulletinBoardClient; import meerkat.comm.CommunicationException; -import meerkat.protobuf.BulletinBoardAPI.*; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; @@ -14,16 +13,18 @@ import java.util.List; /** * Created by Arbel Deutsch Peled on 27-Dec-15. */ -public class MultiServerPostMessageWorker extends MultiServerWorker { +public abstract class MultiServerGenericPostWorker extends MultiServerWorker { - public MultiServerPostMessageWorker(List clients, - int minServers, BulletinBoardMessage payload, int maxRetry, + public MultiServerGenericPostWorker(List clients, + int minServers, T payload, int maxRetry, ClientCallback clientCallback) { super(clients, minServers, payload, maxRetry, clientCallback); } + protected abstract void doPost(SingleServerBulletinBoardClient client, T payload); + /** * This method carries out the actual communication with the servers via HTTP Post * It accesses the servers one by one and tries to post the payload to each in turn @@ -48,7 +49,7 @@ public class MultiServerPostMessageWorker extends MultiServerWorker>{ +public abstract class MultiServerGenericReadWorker extends MultiServerWorker{ - private Iterator clientIterator; + protected Iterator clientIterator; - public MultiServerReadMessagesWorker(List clients, - int minServers, MessageFilterList payload, int maxRetry, - ClientCallback> clientCallback) { + public MultiServerGenericReadWorker(List clients, + int minServers, IN payload, int maxRetry, + ClientCallback clientCallback) { super(clients, true, minServers, payload, maxRetry, clientCallback); // Shuffle clients on creation to balance load @@ -27,6 +26,8 @@ public class MultiServerReadMessagesWorker extends MultiServerWorker msg) { + public void handleCallback(OUT msg) { succeed(msg); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerGetRedundancyWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java similarity index 97% rename from bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerGetRedundancyWorker.java rename to bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java index 4d5e7f2..5675cb8 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/MultiServerGetRedundancyWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java @@ -1,4 +1,4 @@ -package meerkat.bulletinboard.workers; +package meerkat.bulletinboard.workers.multiserver; import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; import meerkat.bulletinboard.MultiServerWorker; diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchDataWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchDataWorker.java new file mode 100644 index 0000000..df21f9f --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchDataWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.bulletinboard.BatchDataContainer; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerPostBatchDataWorker extends MultiServerGenericPostWorker { + + public MultiServerPostBatchDataWorker(List clients, + int minServers, BatchDataContainer payload, int maxRetry, + ClientCallback clientCallback) { + + super(clients, minServers, payload, maxRetry, clientCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, BatchDataContainer payload) { + client.postBatchData(payload.signerId, payload.batchId, payload.batchDataList, payload.startPosition, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java new file mode 100644 index 0000000..7c2f586 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.CompleteBatch; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerPostBatchWorker extends MultiServerGenericPostWorker { + + public MultiServerPostBatchWorker(List clients, + int minServers, CompleteBatch payload, int maxRetry, + ClientCallback clientCallback) { + + super(clients, minServers, payload, maxRetry, clientCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, CompleteBatch payload) { + client.postBatch(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java new file mode 100644 index 0000000..33f9a3c --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerPostMessageWorker extends MultiServerGenericPostWorker { + + public MultiServerPostMessageWorker(List clients, + int minServers, BulletinBoardMessage payload, int maxRetry, + ClientCallback clientCallback) { + + super(clients, minServers, payload, maxRetry, clientCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, BulletinBoardMessage payload) { + client.postMessage(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java new file mode 100644 index 0000000..737c15c --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java @@ -0,0 +1,30 @@ +package meerkat.bulletinboard.workers.multiserver; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.CompleteBatch; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.BatchSpecificationMessage; + +import java.util.List; + + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerReadBatchWorker extends MultiServerGenericReadWorker { + + public MultiServerReadBatchWorker(List clients, + int minServers, BatchSpecificationMessage payload, int maxRetry, + ClientCallback clientCallback) { + + super(clients, minServers, payload, maxRetry, clientCallback); + + } + + @Override + protected void doRead(BatchSpecificationMessage payload, SingleServerBulletinBoardClient client) { + client.readBatch(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java new file mode 100644 index 0000000..b276eab --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java @@ -0,0 +1,29 @@ +package meerkat.bulletinboard.workers.multiserver; + +import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerReadMessagesWorker extends MultiServerGenericReadWorker>{ + + public MultiServerReadMessagesWorker(List clients, + int minServers, MessageFilterList payload, int maxRetry, + ClientCallback> clientCallback) { + + super(clients, minServers, payload, maxRetry, clientCallback); + + } + + @Override + protected void doRead(MessageFilterList payload, SingleServerBulletinBoardClient client) { + client.readMessages(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerBeginBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerBeginBatchWorker.java new file mode 100644 index 0000000..0c1a1a3 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerBeginBatchWorker.java @@ -0,0 +1,17 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; + +import static meerkat.bulletinboard.BulletinBoardConstants.BEGIN_BATCH_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + * Tries to contact server once and perform a post operation + */ +public class SingleServerBeginBatchWorker extends SingleServerGenericPostWorker { + + public SingleServerBeginBatchWorker(String serverAddress, BeginBatchMessage payload, int maxRetry) { + super(serverAddress, BEGIN_BATCH_PATH, payload, maxRetry); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerCloseBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerCloseBatchWorker.java new file mode 100644 index 0000000..ab298a5 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerCloseBatchWorker.java @@ -0,0 +1,17 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.protobuf.BulletinBoardAPI.CloseBatchMessage; + +import static meerkat.bulletinboard.BulletinBoardConstants.CLOSE_BATCH_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + * Tries to contact server once and perform a close batch operation + */ +public class SingleServerCloseBatchWorker extends SingleServerGenericPostWorker { + + public SingleServerCloseBatchWorker(String serverAddress, CloseBatchMessage payload, int maxRetry) { + super(serverAddress, CLOSE_BATCH_PATH, payload, maxRetry); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerPostMessageWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java similarity index 76% rename from bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerPostMessageWorker.java rename to bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java index ee9fd3f..c56af05 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerPostMessageWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java @@ -1,9 +1,8 @@ -package meerkat.bulletinboard.workers; +package meerkat.bulletinboard.workers.singleserver; import meerkat.bulletinboard.SingleServerWorker; import meerkat.comm.CommunicationException; import meerkat.protobuf.BulletinBoardAPI.BoolMsg; -import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; import meerkat.rest.Constants; import javax.ws.rs.ProcessingException; @@ -13,16 +12,18 @@ import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; -import static meerkat.bulletinboard.BulletinBoardConstants.POST_MESSAGE_PATH; /** * Created by Arbel Deutsch Peled on 27-Dec-15. * Tries to contact server once and perform a post operation */ -public class SingleServerPostMessageWorker extends SingleServerWorker { +public class SingleServerGenericPostWorker extends SingleServerWorker { - public SingleServerPostMessageWorker(String serverAddress, BulletinBoardMessage payload, int maxRetry) { + private String subPath; + + public SingleServerGenericPostWorker(String serverAddress, String subPath, T payload, int maxRetry) { super(serverAddress, payload, maxRetry); + this.subPath = subPath; } /** @@ -30,13 +31,13 @@ public class SingleServerPostMessageWorker extends SingleServerWorker { + + public SingleServerPostBatchWorker(String serverAddress, BatchMessage payload, int maxRetry) { + super(serverAddress, POST_BATCH_PATH, payload, maxRetry); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostMessageWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostMessageWorker.java new file mode 100644 index 0000000..454d720 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostMessageWorker.java @@ -0,0 +1,17 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; + +import static meerkat.bulletinboard.BulletinBoardConstants.POST_MESSAGE_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + * Tries to contact server once and perform a post operation + */ +public class SingleServerPostMessageWorker extends SingleServerGenericPostWorker { + + public SingleServerPostMessageWorker(String serverAddress, BulletinBoardMessage payload, int maxRetry) { + super(serverAddress, POST_MESSAGE_PATH, payload, maxRetry); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java new file mode 100644 index 0000000..61556fc --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java @@ -0,0 +1,119 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.bulletinboard.CompleteBatch; +import meerkat.bulletinboard.SingleServerWorker; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.rest.Constants; + +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.GenericType; +import javax.ws.rs.core.Response; +import java.util.List; + +import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; +import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH; +import static meerkat.bulletinboard.BulletinBoardConstants.READ_BATCH_PATH; + +import static meerkat.bulletinboard.BulletinBoardConstants.BATCH_ID_TAG_PREFIX; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class SingleServerReadBatchWorker extends SingleServerWorker { + + public SingleServerReadBatchWorker(String serverAddress, BatchSpecificationMessage payload, int maxRetry) { + super(serverAddress, payload, maxRetry); + } + + /** + * This method carries out the actual communication with the server via HTTP Post + * Upon successful retrieval from the server the method returns the received values + * @return the complete batch as read from the server + * @throws CommunicationException if the server's response is invalid + */ + public CompleteBatch call() throws CommunicationException{ + + CompleteBatch completeBatch = new CompleteBatch(); + + Client client = clientLocal.get(); + + WebTarget webTarget; + Response response; + + // Set filters for the batch message metadata retrieval + + MessageFilterList messageFilterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(BATCH_ID_TAG_PREFIX + String.valueOf(payload.getBatchId())) + .build()) + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.SIGNER_ID) + .setId(payload.getSignerId()) + .build()) + .build(); + + // Send request to Server + + webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); + response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post( + Entity.entity(messageFilterList, Constants.MEDIATYPE_PROTOBUF)); + + // Retrieve answer + + try { + + // If a BulletinBoardMessageList is returned: the read was successful + BulletinBoardMessage metadata = response.readEntity(BulletinBoardMessageList.class).getMessage(0); + + completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder() + .setSignerId(payload.getSignerId()) + .setBatchId(payload.getBatchId()) + .addAllTag(metadata.getMsg().getTagList()) + .build()); + + completeBatch.setSignature(metadata.getSig(0)); + + } catch (ProcessingException | IllegalStateException e) { + + // Read failed + throw new CommunicationException("Could not contact the server"); + + } + finally { + response.close(); + } + + // Get the batch data + + webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_BATCH_PATH); + response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post( + Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF)); + + // Retrieve answer + + try { + + // If a List of BatchData is returned: the read was successful + + completeBatch.appendBatchData(response.readEntity(new GenericType>(){})); + + } catch (ProcessingException | IllegalStateException e) { + + // Read failed + throw new CommunicationException("Could not contact the server"); + + } + finally { + response.close(); + } + + return completeBatch; + + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerReadMessagesWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java similarity index 97% rename from bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerReadMessagesWorker.java rename to bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java index f8975cb..6c09bcc 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/SingleServerReadMessagesWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java @@ -1,4 +1,4 @@ -package meerkat.bulletinboard.workers; +package meerkat.bulletinboard.workers.singleserver; import meerkat.bulletinboard.SingleServerWorker; import meerkat.comm.CommunicationException; diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java index cab9d6d..fe3d2fc 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java @@ -141,6 +141,10 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL } } + @Path(READ_BATCH_PATH) + @POST + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) @Override public List readBatch(BatchSpecificationMessage message) { try { diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java index 00bd2ac..7732fcb 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java @@ -1,7 +1,7 @@ package meerkat.bulletinboard; +import com.google.protobuf.ByteString; import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.protobuf.Crypto.Signature; import java.util.List; @@ -37,36 +37,47 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { /** * This message informs the server about the existence of a new batch message and supplies it with the tags associated with it - * @param signerId is the canonical form for the ID of the sender of this batch - * @param batchId is a unique (per signer) ID for this batch - * @param tagList is a list of tags that belong to the batch message + * @param beginBatchMessage contains the data required to begin the batch + * @param callback is a callback function class for handling results of the operation */ - public void beginBatch(byte[] signerId, int batchId, List tagList, ClientCallback callback); + public void beginBatch(BeginBatchMessage beginBatchMessage, ClientCallback callback); /** * This method posts batch data into an (assumed to be open) batch * It does not close the batch * @param signerId is the canonical form for the ID of the sender of this batch * @param batchId is a unique (per signer) ID for this batch - * @param batchDataList is the (canonically ordered) list of data comprising the entire batch message (not just the portion to be written) + * @param batchDataList is the (canonically ordered) list of data comprising the portion of the batch to be posted * @param startPosition is the location (in the batch) of the first entry in batchDataList * (optionally used to continue interrupted post operations) + * The first position in the batch is position 0 * @param callback is a callback function class for handling results of the operation */ public void postBatchData(byte[] signerId, int batchId, List batchDataList, - int startPosition, ClientCallback callback); + int startPosition, ClientCallback callback); /** - * Overloading of the postBatchData method in which startPosition is set to the default value 0 + * Overloading of the postBatchData method which starts at the first position in the batch */ public void postBatchData(byte[] signerId, int batchId, List batchDataList, ClientCallback callback); + /** + * Overloading of the postBatchData method which uses ByteString + */ + public void postBatchData(ByteString signerId, int batchId, List batchDataList, + int startPosition, ClientCallback callback); + + /** + * Overloading of the postBatchData method which uses ByteString and starts at the first position in the batch + */ + public void postBatchData(ByteString signerId, int batchId, List batchDataList, ClientCallback callback); + /** * Attempts to close a batch message * @param closeBatchMessage contains the data required to close the batch * @param callback is a callback function class for handling results of the operation */ - public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback); + public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback); /** * Check how "safe" a given message is in an asynchronous manner @@ -88,11 +99,10 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { /** * Read a given batch message from the bulletin board - * @param signerId is the ID of the signer (sender) of the batch message - * @param batchId is the unique (per signer) ID of the batch + * @param batchSpecificationMessage contains the data required to specify a single batch instance * @param callback is a callback class for handling the result of the operation */ - public void readBatch(byte[] signerId, int batchId, ClientCallback callback); + public void readBatch(BatchSpecificationMessage batchSpecificationMessage, ClientCallback callback); /** * Subscribes to a notifier that will return any new messages on the server that match the given filters diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java index 0db1d3f..6bfc06f 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java @@ -9,6 +9,7 @@ public interface BulletinBoardConstants { public static final String BULLETIN_BOARD_SERVER_PATH = "/bbserver"; public static final String READ_MESSAGES_PATH = "/readmessages"; + public static final String READ_BATCH_PATH = "/readbatch"; public static final String POST_MESSAGE_PATH = "/postmessage"; public static final String BEGIN_BATCH_PATH = "/beginbatch"; public static final String POST_BATCH_PATH = "/postbatch"; @@ -17,6 +18,6 @@ public interface BulletinBoardConstants { // Other Constants public static final String BATCH_TAG = "@BATCH"; - public static final String BATCH_ID_TAG_PREFIX = "#"; + public static final String BATCH_ID_TAG_PREFIX = "BATCHID#"; } diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java index 8171271..4f25f59 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java @@ -2,7 +2,7 @@ package meerkat.bulletinboard; import com.google.protobuf.Message; import meerkat.crypto.Digest; -import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; +import meerkat.protobuf.BulletinBoardAPI.MessageID; import meerkat.protobuf.BulletinBoardAPI.BatchData; import java.util.List; @@ -36,6 +36,11 @@ public class GenericBatchDigest implements BatchDigest{ return digest.digest(); } + @Override + public MessageID digestAsMessageID() { + return digest.digestAsMessageID(); + } + @Override public void update(Message msg) { digest.update(msg); diff --git a/meerkat-common/src/main/java/meerkat/crypto/Digest.java b/meerkat-common/src/main/java/meerkat/crypto/Digest.java index c72206e..b7d86dc 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/Digest.java +++ b/meerkat-common/src/main/java/meerkat/crypto/Digest.java @@ -1,6 +1,7 @@ package meerkat.crypto; import com.google.protobuf.Message; +import meerkat.protobuf.BulletinBoardAPI.MessageID; import java.security.MessageDigest; @@ -15,6 +16,12 @@ public interface Digest { */ public byte[] digest(); + /** + * Completes the hash computation and returns a MessageID Protobuf as output + * @return + */ + public MessageID digestAsMessageID(); + /** * Updates the digest using the specified message (in serialized wire form) * diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java index 4aac501..a7723ec 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java @@ -3,6 +3,7 @@ package meerkat.crypto.concrete; import com.google.protobuf.ByteString; import com.google.protobuf.Message; import meerkat.crypto.Digest; +import meerkat.protobuf.BulletinBoardAPI.MessageID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,6 +61,11 @@ public class SHA256Digest implements Digest { return hash.digest(); } + @Override + public MessageID digestAsMessageID() { + return MessageID.newBuilder().setID(ByteString.copyFrom(digest())).build(); + } + @Override public void update(Message msg) { From 9a78330e29fb7d432c22e57683a0fce458c37ccf Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Tue, 16 Feb 2016 22:33:52 +0200 Subject: [PATCH 09/12] Working Integration test for Threaded BB Client supporting Batches. Haven't tested subscriptions yet. --- .../bulletinboard/MultiServerWorker.java | 20 +- .../SingleServerBulletinBoardClient.java | 318 ++++++++-- .../ThreadedBulletinBoardClient.java | 36 +- .../MultiServerBeginBatchWorker.java | 6 +- .../MultiServerCloseBatchWorker.java | 6 +- .../MultiServerGenericPostWorker.java | 16 +- .../MultiServerGenericReadWorker.java | 14 +- .../MultiServerGetRedundancyWorker.java | 12 +- .../MultiServerPostBatchDataWorker.java | 6 +- .../MultiServerPostBatchWorker.java | 6 +- .../MultiServerPostMessageWorker.java | 6 +- .../MultiServerReadBatchWorker.java | 6 +- .../MultiServerReadMessagesWorker.java | 6 +- .../SingleServerReadBatchWorker.java | 57 +- .../BulletinBoardClientIntegrationTest.java | 214 ------- ...dedBulletinBoardClientIntegrationTest.java | 541 ++++++++++++++++++ bulletin-board-server/build.gradle | 7 +- .../sqlserver/BulletinBoardSQLServer.java | 30 +- .../sqlserver/H2QueryProvider.java | 3 + .../sqlserver/MySQLQueryProvider.java | 3 + .../sqlserver/SQLiteQueryProvider.java | 3 + .../webapp/BulletinBoardWebApp.java | 6 +- .../AsyncBulletinBoardClient.java | 28 +- .../bulletinboard/BulletinBoardServer.java | 2 +- .../meerkat/bulletinboard/CompleteBatch.java | 50 ++ .../java/meerkat/util/BulletinBoardUtils.java | 65 +++ .../main/proto/meerkat/BulletinBoardAPI.proto | 12 +- 27 files changed, 1076 insertions(+), 403 deletions(-) delete mode 100644 bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java create mode 100644 bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java create mode 100644 meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java index 8db836f..727a922 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java @@ -2,7 +2,7 @@ package meerkat.bulletinboard; import com.google.common.util.concurrent.FutureCallback; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; import java.util.Collections; import java.util.Iterator; @@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicInteger; * This is a general class for handling multi-server work * It utilizes Single Server Clients to perform the actual per-server work */ -public abstract class MultiServerWorker extends BulletinClientWorker implements Runnable, ClientCallback{ +public abstract class MultiServerWorker extends BulletinClientWorker implements Runnable, FutureCallback{ private List clients; @@ -26,7 +26,7 @@ public abstract class MultiServerWorker extends BulletinClientWorker clientCallback; + private FutureCallback futureCallback; /** * Constructor @@ -35,11 +35,11 @@ public abstract class MultiServerWorker extends BulletinClientWorker clients, boolean shuffleClients, int minServers, IN payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { super(payload,maxRetry); @@ -50,7 +50,7 @@ public abstract class MultiServerWorker extends BulletinClientWorker extends BulletinClientWorker clients, int minServers, IN payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - this(clients, false, minServers, payload, maxRetry, clientCallback); + this(clients, false, minServers, payload, maxRetry, futureCallback); } @@ -74,7 +74,7 @@ public abstract class MultiServerWorker extends BulletinClientWorker extends BulletinClientWorker implements FutureCallback { private SingleServerWorker worker; - private ClientCallback clientCallback; + private FutureCallback futureCallback; - public RetryCallback(SingleServerWorker worker, ClientCallback clientCallback) { + public RetryCallback(SingleServerWorker worker, FutureCallback futureCallback) { this.worker = worker; - this.clientCallback = clientCallback; + this.futureCallback = futureCallback; } @Override public void onSuccess(T result) { - clientCallback.handleCallback(result); + futureCallback.onSuccess(result); } @Override @@ -107,19 +114,210 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i scheduleWorker(worker, this); } else { // No more retries: notify caller about failure - clientCallback.handleFailure(t); + futureCallback.onFailure(t); } } } + /** + * This callback ties together all the per-batch-data callbacks into a single callback + * It reports success back to the user only if all of the batch-data were successfully posted + * If any batch-data fails to post: this callback reports failure + */ + class PostBatchDataListCallback implements FutureCallback { - public SingleServerBulletinBoardClient(int threadPoolSize, long failDelayInMilliseconds) { + private FutureCallback callback; + private AtomicInteger batchDataRemaining; + private AtomicBoolean aggregatedResult; + + public PostBatchDataListCallback(int batchDataLength, FutureCallback callback) { + + this.callback = callback; + this.batchDataRemaining = new AtomicInteger(batchDataLength); + this.aggregatedResult = new AtomicBoolean(false); + + } + + @Override + public void onSuccess(Boolean result) { + + if (result){ + this.aggregatedResult.set(true); + } + + if (batchDataRemaining.decrementAndGet() == 0){ + callback.onSuccess(this.aggregatedResult.get()); + } + } + + @Override + public void onFailure(Throwable t) { + + // Notify caller about failure + callback.onFailure(t); + + } + } + + /** + * This callback ties together the different parts of a CompleteBatch as they arrive from the server + * It assembles a CompleteBatch from the parts and sends it to the user if all parts arrived + * If any part fails to arrive: it invokes the onFailure method + */ + class CompleteBatchReadCallback { + + private FutureCallback callback; + + private List batchDataList; + private BulletinBoardMessage batchMessage; + + private AtomicInteger remainingQueries; + private AtomicBoolean failed; + + public CompleteBatchReadCallback(FutureCallback callback) { + + this.callback = callback; + + remainingQueries = new AtomicInteger(2); + failed = new AtomicBoolean(false); + + } + + protected void combineAndReturn() { + + final String[] prefixes = { + BulletinBoardConstants.BATCH_ID_TAG_PREFIX, + BulletinBoardConstants.BATCH_TAG}; + + if (remainingQueries.decrementAndGet() == 0){ + + BeginBatchMessage beginBatchMessage = + BeginBatchMessage.newBuilder() + .setSignerId(batchMessage.getSig(0).getSignerId()) + .setBatchId(Integer.parseInt( + BulletinBoardUtils.findTagWithPrefix(batchMessage, BulletinBoardConstants.BATCH_ID_TAG_PREFIX))) + .addAllTag(BulletinBoardUtils.removePrefixTags(batchMessage, Arrays.asList(prefixes))) + .build(); + callback.onSuccess(new CompleteBatch(beginBatchMessage, batchDataList, batchMessage.getSig(0))); + } + + } + + protected void fail(Throwable t) { + if (failed.compareAndSet(false, true)) { + callback.onFailure(t); + } + } + + /** + * @return a FutureCallback for the Batch Data List that ties to this object + */ + public FutureCallback> asBatchDataListFutureCallback() { + return new FutureCallback>() { + + @Override + public void onSuccess(List result) { + batchDataList = result; + + combineAndReturn(); + } + + @Override + public void onFailure(Throwable t) { + fail(t); + } + + }; + } + + /** + * @return a FutureCallback for the Bulletin Board Message that ties to this object + */ + public FutureCallback> asBulletinBoardMessageListFutureCallback() { + return new FutureCallback>() { + + @Override + public void onSuccess(List result) { + if (result.size() < 1){ + onFailure(new IllegalArgumentException("Server returned empty message list")); + return; + } + + batchMessage = result.get(0); + + combineAndReturn(); + } + + @Override + public void onFailure(Throwable t) { + fail(t); + } + }; + } + + } + + + /** + * Inner class for handling returned values of subscription operations + * This class's methods also ensure continued operation of the subscription + */ + class SubscriptionCallback implements FutureCallback> { + + private SingleServerReadMessagesWorker worker; + private MessageHandler messageHandler; + + private MessageFilterList.Builder filterBuilder; + + public SubscriptionCallback(SingleServerReadMessagesWorker worker, MessageHandler messageHandler) { + this.worker = worker; + this.messageHandler = messageHandler; + filterBuilder = worker.getPayload().toBuilder(); + + } + + @Override + public void onSuccess(List result) { + + // Report new messages to user + messageHandler.handleNewMessages(result); + + // Remove last filter from list (MIN_ENTRY one) + filterBuilder.removeFilter(filterBuilder.getFilterCount() - 1); + + // Add updated MIN_ENTRY filter (entry number is successor of last received entry's number) + filterBuilder.addFilter(MessageFilter.newBuilder() + .setType(FilterType.MIN_ENTRY) + .setEntry(result.get(result.size() - 1).getEntryNum() + 1) + .build()); + + // Create new worker with updated task + worker = new SingleServerReadMessagesWorker(worker.serverAddress, filterBuilder.build(), 1); + + // Schedule the worker + scheduleWorker(worker, this); + + } + + @Override + public void onFailure(Throwable t) { + + // Notify client about failure + fail(); + + // Reschedule exact same task + scheduleWorker(worker, this); + } + } + + public SingleServerBulletinBoardClient(int threadPoolSize, long failDelayInMilliseconds, long subscriptionIntervalInMilliseconds) { executorService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(threadPoolSize)); this.failDelayInMilliseconds = failDelayInMilliseconds; + this.subscriptionIntervalInMilliseconds = subscriptionIntervalInMilliseconds; // Set server error time to a time sufficiently in the past to make new jobs go through lastServerErrorTime = System.currentTimeMillis() - failDelayInMilliseconds; @@ -147,7 +345,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } @Override - public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback) { + public MessageID postMessage(BulletinBoardMessage msg, FutureCallback callback) { // Create worker with redundancy 1 and MAX_RETRIES retries SingleServerPostMessageWorker worker = new SingleServerPostMessageWorker(meerkatDBs.get(0), msg, MAX_RETRIES); @@ -162,18 +360,18 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } - private class PostBatchDataCallback implements ClientCallback { + private class PostBatchDataCallback implements FutureCallback { private CompleteBatch completeBatch; - ClientCallback callback; + FutureCallback callback; - public PostBatchDataCallback(CompleteBatch completeBatch, ClientCallback callback) { + public PostBatchDataCallback(CompleteBatch completeBatch, FutureCallback callback) { this.completeBatch = completeBatch; this.callback = callback; } @Override - public void handleCallback(Boolean msg) { + public void onSuccess(Boolean msg) { closeBatch( CloseBatchMessage.newBuilder() .setBatchId(completeBatch.getBeginBatchMessage().getBatchId()) @@ -185,24 +383,24 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } @Override - public void handleFailure(Throwable t) { - callback.handleFailure(t); + public void onFailure(Throwable t) { + callback.onFailure(t); } } - private class BeginBatchCallback implements ClientCallback { + private class BeginBatchCallback implements FutureCallback { private CompleteBatch completeBatch; - ClientCallback callback; + FutureCallback callback; - public BeginBatchCallback(CompleteBatch completeBatch, ClientCallback callback) { + public BeginBatchCallback(CompleteBatch completeBatch, FutureCallback callback) { this.completeBatch = completeBatch; this.callback = callback; } @Override - public void handleCallback(Boolean msg) { + public void onSuccess(Boolean msg) { postBatchData( completeBatch.getBeginBatchMessage().getSignerId(), @@ -213,13 +411,13 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } @Override - public void handleFailure(Throwable t) { - callback.handleFailure(t); + public void onFailure(Throwable t) { + callback.onFailure(t); } } @Override - public MessageID postBatch(CompleteBatch completeBatch, ClientCallback callback) { + public MessageID postBatch(CompleteBatch completeBatch, FutureCallback callback) { beginBatch( completeBatch.getBeginBatchMessage(), @@ -233,7 +431,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } @Override - public void beginBatch(BeginBatchMessage beginBatchMessage, ClientCallback callback) { + public void beginBatch(BeginBatchMessage beginBatchMessage, FutureCallback callback) { // Create worker with redundancy 1 and MAX_RETRIES retries SingleServerBeginBatchWorker worker = @@ -246,12 +444,16 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i @Override public void postBatchData(ByteString signerId, int batchId, List batchDataList, - int startPosition, ClientCallback callback) { + int startPosition, FutureCallback callback) { BatchMessage.Builder builder = BatchMessage.newBuilder() .setSignerId(signerId) .setBatchId(batchId); + // Create a unified callback to aggregate successful posts + + PostBatchDataListCallback listCallback = new PostBatchDataListCallback(batchDataList.size(), callback); + // Iterate through data list for (BatchData data : batchDataList) { @@ -262,7 +464,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i new SingleServerPostBatchWorker(meerkatDBs.get(0), builder.build(), MAX_RETRIES); // Create worker with redundancy 1 and MAX_RETRIES retries - scheduleWorker(worker, new RetryCallback(worker, callback)); + scheduleWorker(worker, new RetryCallback(worker, listCallback)); // Increment position in batch startPosition++; @@ -271,7 +473,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } @Override - public void postBatchData(ByteString signerId, int batchId, List batchDataList, ClientCallback callback) { + public void postBatchData(ByteString signerId, int batchId, List batchDataList, FutureCallback callback) { postBatchData(signerId, batchId, batchDataList, 0, callback); @@ -279,21 +481,21 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i @Override public void postBatchData(byte[] signerId, int batchId, List batchDataList, - int startPosition, ClientCallback callback) { + int startPosition, FutureCallback callback) { postBatchData(ByteString.copyFrom(signerId), batchId, batchDataList, startPosition, callback); } @Override - public void postBatchData(byte[] signerId, int batchId, List batchDataList, ClientCallback callback) { + public void postBatchData(byte[] signerId, int batchId, List batchDataList, FutureCallback callback) { postBatchData(signerId, batchId, batchDataList, 0, callback); } @Override - public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback) { + public void closeBatch(CloseBatchMessage closeBatchMessage, FutureCallback callback) { // Create worker with redundancy 1 and MAX_RETRIES retries SingleServerCloseBatchWorker worker = @@ -305,7 +507,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } @Override - public void getRedundancy(MessageID id, ClientCallback callback) { + public void getRedundancy(MessageID id, FutureCallback callback) { // Create worker with no retries SingleServerGetRedundancyWorker worker = new SingleServerGetRedundancyWorker(meerkatDBs.get(0), id, 1); @@ -316,7 +518,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } @Override - public void readMessages(MessageFilterList filterList, ClientCallback> callback) { + public void readMessages(MessageFilterList filterList, FutureCallback> callback) { // Create job with no retries SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterList, 1); @@ -327,19 +529,65 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i } @Override - public void readBatch(BatchSpecificationMessage batchSpecificationMessage, ClientCallback callback) { + public void readBatch(BatchSpecificationMessage batchSpecificationMessage, FutureCallback callback) { - // Create job with no retries - SingleServerReadBatchWorker worker = new SingleServerReadBatchWorker(meerkatDBs.get(0), batchSpecificationMessage, 1); + // Create job with no retries for retrieval of the Bulletin Board Message that defines the batch - // Submit job and create callback - scheduleWorker(worker, new RetryCallback(worker, callback)); + MessageFilterList filterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(BulletinBoardConstants.BATCH_TAG) + .build()) + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(BulletinBoardConstants.BATCH_ID_TAG_PREFIX + batchSpecificationMessage.getBatchId()) + .build()) + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.SIGNER_ID) + .setId(batchSpecificationMessage.getSignerId()) + .build()) + .build(); + + SingleServerReadMessagesWorker messageWorker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterList, 1); + + // Create job with no retries for retrieval of the Batch Data List + SingleServerReadBatchWorker batchWorker = new SingleServerReadBatchWorker(meerkatDBs.get(0), batchSpecificationMessage, 1); + + // Create callback that will combine the two worker products + CompleteBatchReadCallback completeBatchReadCallback = new CompleteBatchReadCallback(callback); + + // Submit jobs with wrapped callbacks + scheduleWorker(messageWorker, new RetryCallback(messageWorker, completeBatchReadCallback.asBulletinBoardMessageListFutureCallback())); + scheduleWorker(batchWorker, new RetryCallback(batchWorker, completeBatchReadCallback.asBatchDataListFutureCallback())); } @Override public void subscribe(MessageFilterList filterList, MessageHandler messageHandler) { + // Remove all existing MIN_ENTRY filters and create new one that starts at 0 + + MessageFilterList.Builder filterListBuilder = filterList.toBuilder(); + + Iterator iterator = filterListBuilder.getFilterList().iterator(); + while (iterator.hasNext()) { + MessageFilter filter = iterator.next(); + + if (filter.getType() == FilterType.MIN_ENTRY){ + iterator.remove(); + } + } + filterListBuilder.addFilter(MessageFilter.newBuilder() + .setType(FilterType.MIN_ENTRY) + .setEntry(0) + .build()); + + // Create job with no retries + SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterListBuilder.build(), 1); + + // Submit job and create callback + scheduleWorker(worker, new SubscriptionCallback(worker, messageHandler)); + } @Override diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java index 78d5dba..76e6236 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java @@ -1,5 +1,6 @@ package meerkat.bulletinboard; +import com.google.common.util.concurrent.FutureCallback; import com.google.protobuf.ByteString; import meerkat.bulletinboard.workers.multiserver.*; @@ -37,6 +38,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple private static final int SERVER_THREADPOOL_SIZE = 5; private static final long FAIL_DELAY = 5000; + private static final long SUBSCRIPTION_INTERVAL = 10000; private int minAbsoluteRedundancy; @@ -59,11 +61,16 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple clients = new ArrayList(clientParams.getBulletinBoardAddressCount()); for (String address : clientParams.getBulletinBoardAddressList()){ - SingleServerBulletinBoardClient client = new SingleServerBulletinBoardClient(SERVER_THREADPOOL_SIZE, FAIL_DELAY); + + SingleServerBulletinBoardClient client = + new SingleServerBulletinBoardClient(SERVER_THREADPOOL_SIZE, FAIL_DELAY, SUBSCRIPTION_INTERVAL); + client.init(BulletinBoardClientParams.newBuilder() .addBulletinBoardAddress(address) .build()); + clients.add(client); + } } @@ -76,7 +83,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple * @throws CommunicationException */ @Override - public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback){ + public MessageID postMessage(BulletinBoardMessage msg, FutureCallback callback){ // Create job MultiServerPostMessageWorker worker = @@ -93,7 +100,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple } @Override - public MessageID postBatch(CompleteBatch completeBatch, ClientCallback callback) { + public MessageID postBatch(CompleteBatch completeBatch, FutureCallback callback) { // Create job MultiServerPostBatchWorker worker = @@ -110,7 +117,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple } @Override - public void beginBatch(BeginBatchMessage beginBatchMessage, ClientCallback callback) { + public void beginBatch(BeginBatchMessage beginBatchMessage, FutureCallback callback) { // Create job MultiServerBeginBatchWorker worker = @@ -123,7 +130,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple @Override public void postBatchData(byte[] signerId, int batchId, List batchDataList, - int startPosition, ClientCallback callback) { + int startPosition, FutureCallback callback) { BatchDataContainer batchDataContainer = new BatchDataContainer(signerId, batchId, batchDataList, startPosition); @@ -137,7 +144,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple } @Override - public void postBatchData(byte[] signerId, int batchId, List batchDataList, ClientCallback callback) { + public void postBatchData(byte[] signerId, int batchId, List batchDataList, FutureCallback callback) { postBatchData(signerId, batchId, batchDataList, 0, callback); @@ -145,21 +152,21 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple @Override public void postBatchData(ByteString signerId, int batchId, List batchDataList, - int startPosition, ClientCallback callback) { + int startPosition, FutureCallback callback) { postBatchData(signerId.toByteArray(), batchId, batchDataList, startPosition, callback); } @Override - public void postBatchData(ByteString signerId, int batchId, List batchDataList, ClientCallback callback) { + public void postBatchData(ByteString signerId, int batchId, List batchDataList, FutureCallback callback) { postBatchData(signerId, batchId, batchDataList, 0, callback); } @Override - public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback) { + public void closeBatch(CloseBatchMessage closeBatchMessage, FutureCallback callback) { // Create job MultiServerCloseBatchWorker worker = @@ -177,7 +184,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple * Ignore communication exceptions in specific databases */ @Override - public void getRedundancy(MessageID id, ClientCallback callback) { + public void getRedundancy(MessageID id, FutureCallback callback) { // Create job MultiServerGetRedundancyWorker worker = @@ -194,7 +201,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple * If no operation is successful: return null (NOT blank list) */ @Override - public void readMessages(MessageFilterList filterList, ClientCallback> callback) { + public void readMessages(MessageFilterList filterList, FutureCallback> callback) { // Create job MultiServerReadMessagesWorker worker = @@ -206,7 +213,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple } @Override - public void readBatch(BatchSpecificationMessage batchSpecificationMessage, ClientCallback callback) { + public void readBatch(BatchSpecificationMessage batchSpecificationMessage, FutureCallback callback) { // Create job MultiServerReadBatchWorker worker = @@ -227,6 +234,11 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple super.close(); try { + + for (SingleServerBulletinBoardClient client : clients){ + client.close(); + } + executorService.shutdown(); while (! executorService.isShutdown()) { executorService.awaitTermination(10, TimeUnit.SECONDS); diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java index dc496d7..e0e92bb 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard.workers.multiserver; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; import meerkat.bulletinboard.SingleServerBulletinBoardClient; import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; @@ -13,9 +13,9 @@ public class MultiServerBeginBatchWorker extends MultiServerGenericPostWorker clients, int minServers, BeginBatchMessage payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - super(clients, minServers, payload, maxRetry, clientCallback); + super(clients, minServers, payload, maxRetry, futureCallback); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java index 56b09c5..300440f 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard.workers.multiserver; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; import meerkat.bulletinboard.SingleServerBulletinBoardClient; import meerkat.protobuf.BulletinBoardAPI.CloseBatchMessage; @@ -13,9 +13,9 @@ public class MultiServerCloseBatchWorker extends MultiServerGenericPostWorker clients, int minServers, CloseBatchMessage payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - super(clients, minServers, payload, maxRetry, clientCallback); + super(clients, minServers, payload, maxRetry, futureCallback); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java index 8b62d4e..4ff96b1 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java @@ -1,6 +1,7 @@ package meerkat.bulletinboard.workers.multiserver; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.FutureCallback; import meerkat.bulletinboard.MultiServerWorker; import meerkat.bulletinboard.SingleServerBulletinBoardClient; import meerkat.comm.CommunicationException; @@ -17,9 +18,9 @@ public abstract class MultiServerGenericPostWorker extends MultiServerWorker< public MultiServerGenericPostWorker(List clients, int minServers, T payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - super(clients, minServers, payload, maxRetry, clientCallback); + super(clients, minServers, payload, maxRetry, futureCallback); } @@ -35,11 +36,6 @@ public abstract class MultiServerGenericPostWorker extends MultiServerWorker< */ public void run() { - WebTarget webTarget; - Response response; - - int count = 0; // Used to count number of servers which contain the required message in a GET_REDUNDANCY request. - // Iterate through servers Iterator clientIterator = getClientIterator(); @@ -56,7 +52,7 @@ public abstract class MultiServerGenericPostWorker extends MultiServerWorker< } @Override - public void handleCallback(Boolean result) { + public void onSuccess(Boolean result) { if (result){ if (minServers.decrementAndGet() <= 0){ succeed(Boolean.TRUE); @@ -65,7 +61,7 @@ public abstract class MultiServerGenericPostWorker extends MultiServerWorker< } @Override - public void handleFailure(Throwable t) { + public void onFailure(Throwable t) { if (maxFailedServers.decrementAndGet() < 0){ fail(t); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java index 6f6b425..68fc020 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard.workers.multiserver; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; import meerkat.bulletinboard.MultiServerWorker; import meerkat.bulletinboard.SingleServerBulletinBoardClient; import meerkat.comm.CommunicationException; @@ -18,9 +18,9 @@ public abstract class MultiServerGenericReadWorker extends MultiServerW public MultiServerGenericReadWorker(List clients, int minServers, IN payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - super(clients, true, minServers, payload, maxRetry, clientCallback); // Shuffle clients on creation to balance load + super(clients, true, minServers, payload, maxRetry, futureCallback); // Shuffle clients on creation to balance load clientIterator = getClientIterator(); @@ -41,10 +41,10 @@ public abstract class MultiServerGenericReadWorker extends MultiServerW if (clientIterator.hasNext()) { - // Send request to Server + // Get next server SingleServerBulletinBoardClient client = clientIterator.next(); - // Retrieve answer + // Retrieve answer from server doRead(payload, client); } else { @@ -54,12 +54,12 @@ public abstract class MultiServerGenericReadWorker extends MultiServerW } @Override - public void handleCallback(OUT msg) { + public void onSuccess(OUT msg) { succeed(msg); } @Override - public void handleFailure(Throwable t) { + public void onFailure(Throwable t) { run(); // Retry with next server } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java index 5675cb8..517dbdf 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard.workers.multiserver; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; import meerkat.bulletinboard.MultiServerWorker; import meerkat.bulletinboard.SingleServerBulletinBoardClient; import meerkat.comm.CommunicationException; @@ -20,9 +20,9 @@ public class MultiServerGetRedundancyWorker extends MultiServerWorker clients, int minServers, MessageID payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - super(clients, minServers, payload, maxRetry, clientCallback); // Shuffle clients on creation to balance load + super(clients, minServers, payload, maxRetry, futureCallback); // Shuffle clients on creation to balance load serversContainingMessage = new AtomicInteger(0); totalContactedServers = new AtomicInteger(0); @@ -54,7 +54,7 @@ public class MultiServerGetRedundancyWorker extends MultiServerWorker 0.5) { serversContainingMessage.incrementAndGet(); @@ -67,8 +67,8 @@ public class MultiServerGetRedundancyWorker extends MultiServerWorker clients, int minServers, BatchDataContainer payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - super(clients, minServers, payload, maxRetry, clientCallback); + super(clients, minServers, payload, maxRetry, futureCallback); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java index 7c2f586..1b1f3df 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard.workers.multiserver; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; import meerkat.bulletinboard.CompleteBatch; import meerkat.bulletinboard.SingleServerBulletinBoardClient; @@ -13,9 +13,9 @@ public class MultiServerPostBatchWorker extends MultiServerGenericPostWorker clients, int minServers, CompleteBatch payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - super(clients, minServers, payload, maxRetry, clientCallback); + super(clients, minServers, payload, maxRetry, futureCallback); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java index 33f9a3c..6d3d702 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard.workers.multiserver; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; import meerkat.bulletinboard.SingleServerBulletinBoardClient; import meerkat.protobuf.BulletinBoardAPI.*; @@ -13,9 +13,9 @@ public class MultiServerPostMessageWorker extends MultiServerGenericPostWorker clients, int minServers, BulletinBoardMessage payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - super(clients, minServers, payload, maxRetry, clientCallback); + super(clients, minServers, payload, maxRetry, futureCallback); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java index 737c15c..3d40c8a 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard.workers.multiserver; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; import meerkat.bulletinboard.CompleteBatch; import meerkat.bulletinboard.SingleServerBulletinBoardClient; import meerkat.protobuf.BulletinBoardAPI.BatchSpecificationMessage; @@ -15,9 +15,9 @@ public class MultiServerReadBatchWorker extends MultiServerGenericReadWorker clients, int minServers, BatchSpecificationMessage payload, int maxRetry, - ClientCallback clientCallback) { + FutureCallback futureCallback) { - super(clients, minServers, payload, maxRetry, clientCallback); + super(clients, minServers, payload, maxRetry, futureCallback); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java index b276eab..980d869 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard.workers.multiserver; -import meerkat.bulletinboard.AsyncBulletinBoardClient.ClientCallback; +import com.google.common.util.concurrent.FutureCallback; import meerkat.bulletinboard.SingleServerBulletinBoardClient; import meerkat.protobuf.BulletinBoardAPI.*; @@ -14,9 +14,9 @@ public class MultiServerReadMessagesWorker extends MultiServerGenericReadWorker< public MultiServerReadMessagesWorker(List clients, int minServers, MessageFilterList payload, int maxRetry, - ClientCallback> clientCallback) { + FutureCallback> futureCallback) { - super(clients, minServers, payload, maxRetry, clientCallback); + super(clients, minServers, payload, maxRetry, futureCallback); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java index 61556fc..11fc777 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java @@ -23,7 +23,7 @@ import static meerkat.bulletinboard.BulletinBoardConstants.BATCH_ID_TAG_PREFIX; /** * Created by Arbel Deutsch Peled on 27-Dec-15. */ -public class SingleServerReadBatchWorker extends SingleServerWorker { +public class SingleServerReadBatchWorker extends SingleServerWorker> { public SingleServerReadBatchWorker(String serverAddress, BatchSpecificationMessage payload, int maxRetry) { super(serverAddress, payload, maxRetry); @@ -35,59 +35,13 @@ public class SingleServerReadBatchWorker extends SingleServerWorker call() throws CommunicationException{ Client client = clientLocal.get(); WebTarget webTarget; Response response; - // Set filters for the batch message metadata retrieval - - MessageFilterList messageFilterList = MessageFilterList.newBuilder() - .addFilter(MessageFilter.newBuilder() - .setType(FilterType.TAG) - .setTag(BATCH_ID_TAG_PREFIX + String.valueOf(payload.getBatchId())) - .build()) - .addFilter(MessageFilter.newBuilder() - .setType(FilterType.SIGNER_ID) - .setId(payload.getSignerId()) - .build()) - .build(); - - // Send request to Server - - webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); - response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post( - Entity.entity(messageFilterList, Constants.MEDIATYPE_PROTOBUF)); - - // Retrieve answer - - try { - - // If a BulletinBoardMessageList is returned: the read was successful - BulletinBoardMessage metadata = response.readEntity(BulletinBoardMessageList.class).getMessage(0); - - completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder() - .setSignerId(payload.getSignerId()) - .setBatchId(payload.getBatchId()) - .addAllTag(metadata.getMsg().getTagList()) - .build()); - - completeBatch.setSignature(metadata.getSig(0)); - - } catch (ProcessingException | IllegalStateException e) { - - // Read failed - throw new CommunicationException("Could not contact the server"); - - } - finally { - response.close(); - } - // Get the batch data webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_BATCH_PATH); @@ -98,9 +52,8 @@ public class SingleServerReadBatchWorker extends SingleServerWorker>(){})); + // If a BatchDataList is returned: the read was successful + return response.readEntity(BatchDataList.class).getDataList(); } catch (ProcessingException | IllegalStateException e) { @@ -112,8 +65,6 @@ public class SingleServerReadBatchWorker extends SingleServerWorker thrown; - - protected void genericHandleFailure(Throwable t){ - System.err.println(t.getCause() + " " + t.getMessage()); - thrown.add(t); - jobSemaphore.release(); - } - - private class PostCallback implements ClientCallback{ - - @Override - public void handleCallback(Boolean msg) { - System.err.println("Post operation completed"); - jobSemaphore.release(); - } - - @Override - public void handleFailure(Throwable t) { - genericHandleFailure(t); - } - } - - private class RedundancyCallback implements ClientCallback{ - - private float minRedundancy; - - public RedundancyCallback(float minRedundancy) { - this.minRedundancy = minRedundancy; - } - - @Override - public void handleCallback(Float redundancy) { - System.err.println("Redundancy found is: " + redundancy); - jobSemaphore.release(); - assertThat(redundancy, greaterThanOrEqualTo(minRedundancy)); - } - - @Override - public void handleFailure(Throwable t) { - genericHandleFailure(t); - } - } - - private class ReadCallback implements ClientCallback>{ - - private List expectedMsgList; - - public ReadCallback(List expectedMsgList) { - this.expectedMsgList = expectedMsgList; - } - - @Override - public void handleCallback(List messages) { - - System.err.println(messages); - jobSemaphore.release(); - - BulletinBoardMessageComparator msgComparator = new BulletinBoardMessageComparator(); - - assertThat(messages.size(), is(expectedMsgList.size())); - - Iterator expectedMessageIterator = expectedMsgList.iterator(); - Iterator receivedMessageIterator = messages.iterator(); - - while (expectedMessageIterator.hasNext()) { - assertThat(msgComparator.compare(expectedMessageIterator.next(), receivedMessageIterator.next()), is(0)); - } - - } - - @Override - public void handleFailure(Throwable t) { - genericHandleFailure(t); - } - } - - private AsyncBulletinBoardClient bulletinBoardClient; - - private PostCallback postCallback; - private RedundancyCallback redundancyCallback; - private ReadCallback readCallback; - - private static String PROP_GETTY_URL = "gretty.httpBaseURI"; - private static String DEFAULT_BASE_URL = "http://localhost:8081"; - private static String BASE_URL = System.getProperty(PROP_GETTY_URL, DEFAULT_BASE_URL); - - @Before - public void init(){ - - bulletinBoardClient = new ThreadedBulletinBoardClient(); - - List testDB = new LinkedList(); - testDB.add(BASE_URL); - - bulletinBoardClient.init(BulletinBoardClientParams.newBuilder() - .addBulletinBoardAddress("http://localhost:8081") - .setMinRedundancy((float) 1.0) - .build()); - - postCallback = new PostCallback(); - redundancyCallback = new RedundancyCallback((float) 1.0); - - thrown = new Vector<>(); - jobSemaphore = new Semaphore(0); - - } - - @Test - public void postTest() { - - byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4}; - byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14}; - byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24}; - byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0}; - - BulletinBoardMessage msg; - - MessageFilterList filterList; - List msgList; - - MessageID messageID; - - Comparator msgComparator = new BulletinBoardMessageComparator(); - - msg = BulletinBoardMessage.newBuilder() - .setMsg(UnsignedBulletinBoardMessage.newBuilder() - .addTag("Signature") - .addTag("Trustee") - .setData(ByteString.copyFrom(b1)) - .build()) - .addSig(Crypto.Signature.newBuilder() - .setType(Crypto.SignatureType.DSA) - .setData(ByteString.copyFrom(b2)) - .setSignerId(ByteString.copyFrom(b3)) - .build()) - .addSig(Crypto.Signature.newBuilder() - .setType(Crypto.SignatureType.ECDSA) - .setData(ByteString.copyFrom(b3)) - .setSignerId(ByteString.copyFrom(b2)) - .build()) - .build(); - - messageID = bulletinBoardClient.postMessage(msg,postCallback); - - try { - jobSemaphore.acquire(); - } catch (InterruptedException e) { - System.err.println(e.getCause() + " " + e.getMessage()); - } - - bulletinBoardClient.getRedundancy(messageID,redundancyCallback); - - filterList = MessageFilterList.newBuilder() - .addFilter( - MessageFilter.newBuilder() - .setType(FilterType.TAG) - .setTag("Signature") - .build() - ) - .addFilter( - MessageFilter.newBuilder() - .setType(FilterType.TAG) - .setTag("Trustee") - .build() - ) - .build(); - - msgList = new LinkedList(); - msgList.add(msg); - - readCallback = new ReadCallback(msgList); - - bulletinBoardClient.readMessages(filterList, readCallback); - try { - jobSemaphore.acquire(2); - } catch (InterruptedException e) { - System.err.println(e.getCause() + " " + e.getMessage()); - } - - bulletinBoardClient.close(); - - if (thrown.size() > 0) { - assert false; - } - - } - -} diff --git a/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java b/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java new file mode 100644 index 0000000..c266086 --- /dev/null +++ b/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java @@ -0,0 +1,541 @@ +import com.google.common.util.concurrent.FutureCallback; +import com.google.protobuf.ByteString; +import meerkat.bulletinboard.AsyncBulletinBoardClient; +import meerkat.bulletinboard.CompleteBatch; +import meerkat.bulletinboard.GenericBatchDigitalSignature; +import meerkat.bulletinboard.ThreadedBulletinBoardClient; +import meerkat.comm.CommunicationException; +import meerkat.crypto.concrete.ECDSASignature; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Crypto; + +import meerkat.protobuf.Voting.*; +import meerkat.util.BulletinBoardMessageComparator; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.number.OrderingComparison.*; + +import java.io.IOException; +import java.io.InputStream; +import java.security.*; +import java.security.cert.CertificateException; +import java.util.*; +import java.util.concurrent.Semaphore; + +/** + * Created by Arbel Deutsch Peled on 05-Dec-15. + */ +public class ThreadedBulletinBoardClientIntegrationTest { + + // Signature resources + + private GenericBatchDigitalSignature signers[]; + private ByteString[] signerIDs; + + private static String KEYFILE_EXAMPLE = "/certs/enduser-certs/user1-key-with-password-secret.p12"; + private static String KEYFILE_EXAMPLE3 = "/certs/enduser-certs/user3-key-with-password-shh.p12"; + + private static String KEYFILE_PASSWORD1 = "secret"; + private static String KEYFILE_PASSWORD3 = "shh"; + + public static String CERT1_PEM_EXAMPLE = "/certs/enduser-certs/user1.crt"; + public static String CERT3_PEM_EXAMPLE = "/certs/enduser-certs/user3.crt"; + + // Server data + + private static String PROP_GETTY_URL = "gretty.httpBaseURI"; + private static String DEFAULT_BASE_URL = "http://localhost:8081"; + private static String BASE_URL = System.getProperty(PROP_GETTY_URL, DEFAULT_BASE_URL); + + // Client and callbacks + + private AsyncBulletinBoardClient bulletinBoardClient; + + private PostCallback postCallback; + private PostCallback failPostCallback = new PostCallback(true,false); + + private RedundancyCallback redundancyCallback; + private ReadCallback readCallback; + private ReadBatchCallback readBatchCallback; + + // Sync and misc + + private Semaphore jobSemaphore; + private Vector thrown; + private Random random; + + // Constructor + + public ThreadedBulletinBoardClientIntegrationTest(){ + + signers = new GenericBatchDigitalSignature[2]; + signerIDs = new ByteString[signers.length]; + signers[0] = new GenericBatchDigitalSignature(new ECDSASignature()); + signers[1] = new GenericBatchDigitalSignature(new ECDSASignature()); + + InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE); + char[] password = KEYFILE_PASSWORD1.toCharArray(); + + KeyStore.Builder keyStoreBuilder = null; + try { + keyStoreBuilder = signers[0].getPKCS12KeyStoreBuilder(keyStream, password); + + signers[0].loadSigningCertificate(keyStoreBuilder); + + signers[0].loadVerificationCertificates(getClass().getResourceAsStream(CERT1_PEM_EXAMPLE)); + + keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE3); + password = KEYFILE_PASSWORD3.toCharArray(); + + keyStoreBuilder = signers[1].getPKCS12KeyStoreBuilder(keyStream, password); + signers[1].loadSigningCertificate(keyStoreBuilder); + + signers[1].loadVerificationCertificates(getClass().getResourceAsStream(CERT3_PEM_EXAMPLE)); + + for (int i = 0 ; i < signers.length ; i++) { + signerIDs[i] = signers[i].getSignerID(); + } + + } catch (IOException e) { + System.err.println("Failed reading from signature file " + e.getMessage()); + fail("Failed reading from signature file " + e.getMessage()); + } catch (CertificateException e) { + System.err.println("Failed reading certificate " + e.getMessage()); + fail("Failed reading certificate " + e.getMessage()); + } catch (KeyStoreException e) { + System.err.println("Failed reading keystore " + e.getMessage()); + fail("Failed reading keystore " + e.getMessage()); + } catch (NoSuchAlgorithmException e) { + System.err.println("Couldn't find signing algorithm " + e.getMessage()); + fail("Couldn't find signing algorithm " + e.getMessage()); + } catch (UnrecoverableKeyException e) { + System.err.println("Couldn't find signing key " + e.getMessage()); + fail("Couldn't find signing key " + e.getMessage()); + } + + } + + // Callback definitions + + protected void genericHandleFailure(Throwable t){ + System.err.println(t.getCause() + " " + t.getMessage()); + thrown.add(t); + jobSemaphore.release(); + } + + private class PostCallback implements FutureCallback{ + + private boolean isAssert; + private boolean assertValue; + + public PostCallback() { + this(false); + } + + public PostCallback(boolean isAssert) { + this(isAssert,true); + } + + public PostCallback(boolean isAssert, boolean assertValue) { + this.isAssert = isAssert; + this.assertValue = assertValue; + } + + @Override + public void onSuccess(Boolean msg) { + System.err.println("Post operation completed"); + jobSemaphore.release(); + //TODO: Change Assert mechanism to exception one + if (isAssert) { + if (assertValue) { + assertThat("Post operation failed", msg, is(Boolean.TRUE)); + } else { + assertThat("Post operation succeeded unexpectedly", msg, is(Boolean.FALSE)); + } + } + } + + @Override + public void onFailure(Throwable t) { + genericHandleFailure(t); + } + } + + private class RedundancyCallback implements FutureCallback{ + + private float minRedundancy; + + public RedundancyCallback(float minRedundancy) { + this.minRedundancy = minRedundancy; + } + + @Override + public void onSuccess(Float redundancy) { + System.err.println("Redundancy found is: " + redundancy); + jobSemaphore.release(); + assertThat(redundancy, greaterThanOrEqualTo(minRedundancy)); + } + + @Override + public void onFailure(Throwable t) { + genericHandleFailure(t); + } + } + + private class ReadCallback implements FutureCallback>{ + + private List expectedMsgList; + + public ReadCallback(List expectedMsgList) { + this.expectedMsgList = expectedMsgList; + } + + @Override + public void onSuccess(List messages) { + + System.err.println(messages); + jobSemaphore.release(); + + BulletinBoardMessageComparator msgComparator = new BulletinBoardMessageComparator(); + + assertThat(messages.size(), is(expectedMsgList.size())); + + Iterator expectedMessageIterator = expectedMsgList.iterator(); + Iterator receivedMessageIterator = messages.iterator(); + + while (expectedMessageIterator.hasNext()) { + assertThat(msgComparator.compare(expectedMessageIterator.next(), receivedMessageIterator.next()), is(0)); + } + + } + + @Override + public void onFailure(Throwable t) { + genericHandleFailure(t); + } + } + + private class ReadBatchCallback implements FutureCallback { + + private CompleteBatch expectedBatch; + + public ReadBatchCallback(CompleteBatch expectedBatch) { + this.expectedBatch = expectedBatch; + } + + @Override + public void onSuccess(CompleteBatch batch) { + + System.err.println(batch); + jobSemaphore.release(); + + assertThat("Batch returned is incorrect", batch, is(equalTo(expectedBatch))); + + } + + @Override + public void onFailure(Throwable t) { + genericHandleFailure(t); + } + } + + // Randomness generators + + private byte randomByte(){ + return (byte) random.nextInt(); + } + + private byte[] randomByteArray(int length) { + + byte[] randomBytes = new byte[length]; + + for (int i = 0; i < length ; i++){ + randomBytes[i] = randomByte(); + } + + return randomBytes; + + } + + private CompleteBatch createRandomBatch(int signer, int batchId, int length) throws SignatureException { + + CompleteBatch completeBatch = new CompleteBatch(); + + // Create data + + completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder() + .setSignerId(signerIDs[signer]) + .setBatchId(batchId) + .addTag("Test") + .build()); + + for (int i = 0 ; i < length ; i++){ + + BatchData batchData = BatchData.newBuilder() + .setData(ByteString.copyFrom(randomByteArray(i))) + .build(); + + completeBatch.appendBatchData(batchData); + + } + + signers[signer].updateContent(completeBatch); + + completeBatch.setSignature(signers[signer].sign()); + + return completeBatch; + + } + + // Test methods + + /** + * Takes care of initializing the client and the test resources + */ + @Before + public void init(){ + + bulletinBoardClient = new ThreadedBulletinBoardClient(); + + random = new Random(0); // We use insecure randomness in tests for repeatability + + List testDB = new LinkedList(); + testDB.add(BASE_URL); + + bulletinBoardClient.init(BulletinBoardClientParams.newBuilder() + .addBulletinBoardAddress("http://localhost:8081") + .setMinRedundancy((float) 1.0) + .build()); + + postCallback = new PostCallback(); + redundancyCallback = new RedundancyCallback((float) 1.0); + + thrown = new Vector<>(); + jobSemaphore = new Semaphore(0); + + } + + /** + * Closes the client and makes sure the test fails when an exception occurred in a separate thread + */ + + @After + public void close() { + + bulletinBoardClient.close(); + + if (thrown.size() > 0) { + assert false; + } + + } + + /** + * Tests the standard post, redundancy and read methods + */ + @Test + public void postTest() { + + byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4}; + byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14}; + byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24}; + byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0}; + + BulletinBoardMessage msg; + + MessageFilterList filterList; + List msgList; + + MessageID messageID; + + Comparator msgComparator = new BulletinBoardMessageComparator(); + + msg = BulletinBoardMessage.newBuilder() + .setMsg(UnsignedBulletinBoardMessage.newBuilder() + .addTag("Signature") + .addTag("Trustee") + .setData(ByteString.copyFrom(b1)) + .build()) + .addSig(Crypto.Signature.newBuilder() + .setType(Crypto.SignatureType.DSA) + .setData(ByteString.copyFrom(b2)) + .setSignerId(ByteString.copyFrom(b3)) + .build()) + .addSig(Crypto.Signature.newBuilder() + .setType(Crypto.SignatureType.ECDSA) + .setData(ByteString.copyFrom(b3)) + .setSignerId(ByteString.copyFrom(b2)) + .build()) + .build(); + + messageID = bulletinBoardClient.postMessage(msg,postCallback); + + try { + jobSemaphore.acquire(); + } catch (InterruptedException e) { + System.err.println(e.getCause() + " " + e.getMessage()); + } + + bulletinBoardClient.getRedundancy(messageID,redundancyCallback); + + filterList = MessageFilterList.newBuilder() + .addFilter( + MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag("Signature") + .build() + ) + .addFilter( + MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag("Trustee") + .build() + ) + .build(); + + msgList = new LinkedList(); + msgList.add(msg); + + readCallback = new ReadCallback(msgList); + + bulletinBoardClient.readMessages(filterList, readCallback); + try { + jobSemaphore.acquire(2); + } catch (InterruptedException e) { + System.err.println(e.getCause() + " " + e.getMessage()); + } + + } + + /** + * Tests posting a batch by parts + * Also tests not being able to post to a closed batch + * @throws CommunicationException, SignatureException, InterruptedException + */ + @Test + public void testBatchPost() throws CommunicationException, SignatureException, InterruptedException { + + final int SIGNER = 1; + final int BATCH_ID = 100; + final int BATCH_LENGTH = 100; + + CompleteBatch completeBatch = createRandomBatch(SIGNER, BATCH_ID, BATCH_LENGTH); + + // Begin batch + + bulletinBoardClient.beginBatch(completeBatch.getBeginBatchMessage(), postCallback); + + jobSemaphore.acquire(); + + // Post data + + bulletinBoardClient.postBatchData(signerIDs[SIGNER], BATCH_ID, completeBatch.getBatchDataList(), postCallback); + + jobSemaphore.acquire(); + + // Close batch + + CloseBatchMessage closeBatchMessage = CloseBatchMessage.newBuilder() + .setBatchId(BATCH_ID) + .setBatchLength(BATCH_LENGTH) + .setSig(completeBatch.getSignature()) + .build(); + + bulletinBoardClient.closeBatch(closeBatchMessage, postCallback); + + jobSemaphore.acquire(); + + // Attempt to open batch again + + bulletinBoardClient.beginBatch(completeBatch.getBeginBatchMessage(), failPostCallback); + + // Attempt to add batch data + + bulletinBoardClient.postBatchData(signerIDs[SIGNER], BATCH_ID, completeBatch.getBatchDataList(), failPostCallback); + + jobSemaphore.acquire(2); + + // Read batch data + + BatchSpecificationMessage batchSpecificationMessage = + BatchSpecificationMessage.newBuilder() + .setSignerId(signerIDs[SIGNER]) + .setBatchId(BATCH_ID) + .setStartPosition(0) + .build(); + + readBatchCallback = new ReadBatchCallback(completeBatch); + + bulletinBoardClient.readBatch(batchSpecificationMessage, readBatchCallback); + + jobSemaphore.acquire(); + + } + + /** + * Posts a complete batch message + * Checks reading od the message + * @throws CommunicationException, SignatureException, InterruptedException + */ + @Test + public void testCompleteBatchPost() throws CommunicationException, SignatureException, InterruptedException { + + final int SIGNER = 0; + final int BATCH_ID = 101; + final int BATCH_LENGTH = 50; + + // Post batch + + CompleteBatch completeBatch = createRandomBatch(SIGNER, BATCH_ID, BATCH_LENGTH); + + bulletinBoardClient.postBatch(completeBatch,postCallback); + + jobSemaphore.acquire(); + + // Read batch + + BatchSpecificationMessage batchSpecificationMessage = + BatchSpecificationMessage.newBuilder() + .setSignerId(signerIDs[SIGNER]) + .setBatchId(BATCH_ID) + .setStartPosition(0) + .build(); + + readBatchCallback = new ReadBatchCallback(completeBatch); + + bulletinBoardClient.readBatch(batchSpecificationMessage, readBatchCallback); + + jobSemaphore.acquire(); + + } + + /** + * Tests that an unopened batch cannot be closed + * @throws CommunicationException, InterruptedException + */ + @Test + public void testInvalidBatchClose() throws CommunicationException, InterruptedException { + + final int NON_EXISTENT_BATCH_ID = 999; + + CloseBatchMessage closeBatchMessage = + CloseBatchMessage.newBuilder() + .setBatchId(NON_EXISTENT_BATCH_ID) + .setBatchLength(1) + .setSig(Crypto.Signature.getDefaultInstance()) + .build(); + + // Try to close the (unopened) batch; + + bulletinBoardClient.closeBatch(closeBatchMessage, failPostCallback); + + jobSemaphore.acquire(); + + } + +} diff --git a/bulletin-board-server/build.gradle b/bulletin-board-server/build.gradle index 21604d1..59045a2 100644 --- a/bulletin-board-server/build.gradle +++ b/bulletin-board-server/build.gradle @@ -48,7 +48,7 @@ dependencies { // JDBC connections compile 'org.springframework:spring-jdbc:4.2.+' - compile 'org.xerial:sqlite-jdbc:3.7.+' + compile 'org.xerial:sqlite-jdbc:3.8.+' compile 'mysql:mysql-connector-java:5.1.+' compile 'com.h2database:h2:1.0.+' @@ -89,6 +89,11 @@ task h2Test(type: Test) { outputs.upToDateWhen { false } } +task liteTest(type: Test) { + include '**/*SQLite*Test*' + outputs.upToDateWhen { false } +} + task dbTest(type: Test) { include '**/*H2*Test*' include '**/*MySQL*Test*' diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java index 5b4fac9..a6313b6 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java @@ -177,8 +177,9 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ case MSG_ID: return MSG_ID; - case EXACT_ENTRY: // Go through - case MAX_ENTRY: + case EXACT_ENTRY: // Go through + case MAX_ENTRY: // Go through + case MIN_ENTRY: return ENTRY_NUM; case SIGNER_ID: @@ -253,12 +254,13 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ switch (messageFilter.getType()) { - case MSG_ID: // Go through + case MSG_ID: // Go through case SIGNER_ID: return messageFilter.getId().toByteArray(); - case EXACT_ENTRY: // Go through - case MAX_ENTRY: + case EXACT_ENTRY: // Go through + case MAX_ENTRY: // Go through + case MIN_ENTRY: return messageFilter.getEntry(); case TAG: @@ -653,7 +655,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ for (int i=0 ; i < tags.length ; i++) { namedParameters[i] = new MapSqlParameterSource(); - namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(0),message.getSignerId()); + namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(0),message.getSignerId().toByteArray()); namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(1),message.getBatchId()); namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(2),tags[i]); } @@ -675,7 +677,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ String sql = sqlQueryProvider.getSQLString(QueryType.INSERT_BATCH_DATA); MapSqlParameterSource namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(0),batchMessage.getSignerId()); + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(0),batchMessage.getSignerId().toByteArray()); namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(1),batchMessage.getBatchId()); namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(2),batchMessage.getSerialNum()); namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(3),batchMessage.getData().toByteArray()); @@ -700,7 +702,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ MapSqlParameterSource namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(0),signerId); + namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(0),signerId.toByteArray()); namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(1),batchId); List lengthResult = jdbcTemplate.query(sql, namedParameters, new LongMapper()); @@ -733,7 +735,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA); namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),signerId); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),signerId.toByteArray()); namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),batchId); namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),0); // Read from the beginning @@ -775,7 +777,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ sql = sqlQueryProvider.getSQLString(QueryType.REMOVE_BATCH_TAGS); namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.REMOVE_BATCH_TAGS.getParamName(0), signerId); + namedParameters.addValue(QueryType.REMOVE_BATCH_TAGS.getParamName(0), signerId.toByteArray()); namedParameters.addValue(QueryType.REMOVE_BATCH_TAGS.getParamName(1), batchId); jdbcTemplate.update(sql, namedParameters); @@ -786,7 +788,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } @Override - public List readBatch(BatchSpecificationMessage message) throws CommunicationException, IllegalArgumentException{ + public BatchDataList readBatch(BatchSpecificationMessage message) throws CommunicationException, IllegalArgumentException{ // Check that batch is closed if (!isBatchClosed(message.getSignerId(), message.getBatchId())) { @@ -796,11 +798,13 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ String sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA); MapSqlParameterSource namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),message.getSignerId()); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),message.getSignerId().toByteArray()); namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),message.getBatchId()); namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),message.getStartPosition()); - return jdbcTemplate.query(sql, namedParameters, new BatchDataMapper()); + return BatchDataList.newBuilder() + .addAllData(jdbcTemplate.query(sql, namedParameters, new BatchDataMapper())) + .build(); } @Override diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java index 14bf9e2..659d9c3 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java @@ -134,6 +134,8 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider return "MsgTable.EntryNum = :EntryNum" + serialString; case MAX_ENTRY: return "MsgTable.EntryNum <= :EntryNum" + serialString; + case MIN_ENTRY: + return "MsgTable.EntryNum >= :EntryNum" + serialString; case MAX_MESSAGES: return "LIMIT :Limit" + serialString; case MSG_ID: @@ -157,6 +159,7 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider switch(filterType) { case EXACT_ENTRY: // Go through case MAX_ENTRY: // Go through + case MIN_ENTRY: // Go through case MAX_MESSAGES: return "INT"; diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java index c8357a3..e3bdef0 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java @@ -151,6 +151,8 @@ public class MySQLQueryProvider implements SQLQueryProvider { return "MsgTable.EntryNum = :EntryNum" + serialString; case MAX_ENTRY: return "MsgTable.EntryNum <= :EntryNum" + serialString; + case MIN_ENTRY: + return "MsgTable.EntryNum >= :EntryNum" + serialString; case MAX_MESSAGES: return "LIMIT :Limit" + serialString; case MSG_ID: @@ -174,6 +176,7 @@ public class MySQLQueryProvider implements SQLQueryProvider { switch(filterType) { case EXACT_ENTRY: // Go through case MAX_ENTRY: // Go through + case MIN_ENTRY: // Go through case MAX_MESSAGES: return "INT"; diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java index d796789..b581b87 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java @@ -54,6 +54,8 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi return "MsgTable.EntryNum = :EntryNum" + serialString; case MAX_ENTRY: return "MsgTable.EntryNum <= :EntryNum" + serialString; + case MIN_ENTRY: + return "MsgTable.EntryNum <= :EntryNum" + serialString; case MAX_MESSAGES: return "LIMIT = :Limit" + serialString; case MSG_ID: @@ -77,6 +79,7 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi switch(filterType) { case EXACT_ENTRY: // Go through case MAX_ENTRY: // Go through + case MIN_ENTRY: // Go through case MAX_MESSAGES: return "INTEGER"; diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java index fe3d2fc..766af19 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java @@ -106,6 +106,7 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL @Override public BoolMsg beginBatch(BeginBatchMessage message) { try { + init(); return bulletinBoard.beginBatch(message); } catch (CommunicationException e) { System.err.println(e.getMessage()); @@ -120,6 +121,7 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL @Override public BoolMsg postBatchMessage(BatchMessage batchMessage) { try { + init(); return bulletinBoard.postBatchMessage(batchMessage); } catch (CommunicationException e) { System.err.println(e.getMessage()); @@ -134,6 +136,7 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL @Override public BoolMsg closeBatchMessage(CloseBatchMessage message) { try { + init(); return bulletinBoard.closeBatchMessage(message); } catch (CommunicationException e) { System.err.println(e.getMessage()); @@ -146,8 +149,9 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL @Consumes(MEDIATYPE_PROTOBUF) @Produces(MEDIATYPE_PROTOBUF) @Override - public List readBatch(BatchSpecificationMessage message) { + public BatchDataList readBatch(BatchSpecificationMessage message) { try { + init(); return bulletinBoard.readBatch(message); } catch (CommunicationException | IllegalArgumentException e) { System.err.println(e.getMessage()); diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java index 7732fcb..c6e330c 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java @@ -1,5 +1,6 @@ package meerkat.bulletinboard; +import com.google.common.util.concurrent.FutureCallback; import com.google.protobuf.ByteString; import meerkat.protobuf.BulletinBoardAPI.*; @@ -10,11 +11,6 @@ import java.util.List; */ public interface AsyncBulletinBoardClient extends BulletinBoardClient { - public interface ClientCallback { - void handleCallback(T msg); - void handleFailure(Throwable t); - } - public interface MessageHandler { void handleNewMessages(List messageList); } @@ -25,7 +21,7 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param callback is a class containing methods to handle the result of the operation * @return a unique message ID for the message, that can be later used to retrieve the batch */ - public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback); + public MessageID postMessage(BulletinBoardMessage msg, FutureCallback callback); /** * Perform an end-to-end post of a signed batch message @@ -33,14 +29,14 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param callback is a class containing methods to handle the result of the operation * @return a unique identifier for the batch message */ - public MessageID postBatch(CompleteBatch completeBatch, ClientCallback callback); + public MessageID postBatch(CompleteBatch completeBatch, FutureCallback callback); /** * This message informs the server about the existence of a new batch message and supplies it with the tags associated with it * @param beginBatchMessage contains the data required to begin the batch * @param callback is a callback function class for handling results of the operation */ - public void beginBatch(BeginBatchMessage beginBatchMessage, ClientCallback callback); + public void beginBatch(BeginBatchMessage beginBatchMessage, FutureCallback callback); /** * This method posts batch data into an (assumed to be open) batch @@ -54,30 +50,30 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param callback is a callback function class for handling results of the operation */ public void postBatchData(byte[] signerId, int batchId, List batchDataList, - int startPosition, ClientCallback callback); + int startPosition, FutureCallback callback); /** * Overloading of the postBatchData method which starts at the first position in the batch */ - public void postBatchData(byte[] signerId, int batchId, List batchDataList, ClientCallback callback); + public void postBatchData(byte[] signerId, int batchId, List batchDataList, FutureCallback callback); /** * Overloading of the postBatchData method which uses ByteString */ public void postBatchData(ByteString signerId, int batchId, List batchDataList, - int startPosition, ClientCallback callback); + int startPosition, FutureCallback callback); /** * Overloading of the postBatchData method which uses ByteString and starts at the first position in the batch */ - public void postBatchData(ByteString signerId, int batchId, List batchDataList, ClientCallback callback); + public void postBatchData(ByteString signerId, int batchId, List batchDataList, FutureCallback callback); /** * Attempts to close a batch message * @param closeBatchMessage contains the data required to close the batch * @param callback is a callback function class for handling results of the operation */ - public void closeBatch(CloseBatchMessage closeBatchMessage, ClientCallback callback); + public void closeBatch(CloseBatchMessage closeBatchMessage, FutureCallback callback); /** * Check how "safe" a given message is in an asynchronous manner @@ -85,7 +81,7 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param id is the unique message identifier for retrieval * @param callback is a callback function class for handling results of the operation */ - public void getRedundancy(MessageID id, ClientCallback callback); + public void getRedundancy(MessageID id, FutureCallback callback); /** * Read all messages posted matching the given filter in an asynchronous manner @@ -95,14 +91,14 @@ public interface AsyncBulletinBoardClient extends BulletinBoardClient { * @param filterList return only messages that match the filters (null means no filtering). * @param callback is a callback function class for handling results of the operation */ - public void readMessages(MessageFilterList filterList, ClientCallback> callback); + public void readMessages(MessageFilterList filterList, FutureCallback> callback); /** * Read a given batch message from the bulletin board * @param batchSpecificationMessage contains the data required to specify a single batch instance * @param callback is a callback class for handling the result of the operation */ - public void readBatch(BatchSpecificationMessage batchSpecificationMessage, ClientCallback callback); + public void readBatch(BatchSpecificationMessage batchSpecificationMessage, FutureCallback callback); /** * Subscribes to a notifier that will return any new messages on the server that match the given filters diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java index 70721a7..cbf06ff 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java @@ -75,7 +75,7 @@ public interface BulletinBoardServer{ * @throws CommunicationException on DB connection error * @throws IllegalArgumentException if message does not specify a batch */ - public List readBatch(BatchSpecificationMessage message) throws CommunicationException, IllegalArgumentException; + public BatchDataList readBatch(BatchSpecificationMessage message) throws CommunicationException, IllegalArgumentException; /** * This method closes the connection to the DB diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java index 19c57e3..227c7ca 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java @@ -2,6 +2,7 @@ package meerkat.bulletinboard; import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.protobuf.Crypto.*; +import meerkat.util.BulletinBoardMessageComparator; import java.util.LinkedList; import java.util.List; @@ -48,6 +49,14 @@ public class CompleteBatch { return signature; } + public CloseBatchMessage getCloseBatchMessage() { + return CloseBatchMessage.newBuilder() + .setBatchId(getBeginBatchMessage().getBatchId()) + .setBatchLength(getBatchDataList().size()) + .setSig(getSignature()) + .build(); + } + public void setBeginBatchMessage(BeginBatchMessage beginBatchMessage) { this.beginBatchMessage = beginBatchMessage; } @@ -64,4 +73,45 @@ public class CompleteBatch { signature = newSignature; } + @Override + public boolean equals(Object other) { + + if (!(other instanceof CompleteBatch)) { + return false; + } + + CompleteBatch otherBatch = (CompleteBatch) other; + + boolean result = true; + + if (beginBatchMessage == null) { + if (otherBatch.getBeginBatchMessage() != null) + return false; + } else { + result = result && beginBatchMessage.equals(otherBatch.getBeginBatchMessage()); + } + + if (batchDataList == null) { + if (otherBatch.getBatchDataList() != null) + return false; + } else { + result = result && batchDataList.equals(otherBatch.getBatchDataList()); + } + + if (signature == null) { + if (otherBatch.getSignature() != null) + return false; + } else { + result = result && signature.equals(otherBatch.getSignature()); + } + + return result; + + } + + @Override + public String toString() { + return "Batch " + beginBatchMessage.getSignerId().toString() + ":" + beginBatchMessage.getBatchId(); + } + } diff --git a/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java b/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java new file mode 100644 index 0000000..b19bab3 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java @@ -0,0 +1,65 @@ +package meerkat.util; + +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.LinkedList; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 16-Feb-16. + */ +public class BulletinBoardUtils { + + /** + * Searches the tags in the message for one that begins with given prefix + * @param message is the message to search + * @param prefix is the given prefix + * @return the tag without the prefix, if found, or null if not found + */ + public static String findTagWithPrefix(BulletinBoardMessage message, String prefix) { + + for (String tag : message.getMsg().getTagList()){ + if (tag.startsWith(prefix)) { + return tag.substring(prefix.length()); + } + } + + return null; + + } + + /** + * Searches the tags in a message for tags that do not contain a given list of prefixes + * @param message is the message to search + * @param prefixes is the list of prefixes + * @return a list of the tags that do *not* contain any of the given prefixes + */ + public static List removePrefixTags(BulletinBoardMessage message, Iterable prefixes) { + + if (prefixes == null) + return message.getMsg().getTagList(); + + List result = new LinkedList<>(); + + for (String tag : message.getMsg().getTagList()){ + + boolean found = false; + + for (String prefix : prefixes){ + if (tag.startsWith(prefix)){ + found = true; + break; + } + } + + if (!found) { + result.add(tag); + } + + } + + return result; + + } + +} diff --git a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto index 2ae4068..b86debd 100644 --- a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto +++ b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto @@ -50,13 +50,14 @@ enum FilterType { MSG_ID = 0; // Match exact message ID EXACT_ENTRY = 1; // Match exact entry number in database (chronological) MAX_ENTRY = 2; // Find all entries in database up to specified entry number (chronological) - SIGNER_ID = 3; // Find all entries in database that correspond to specific signature (signer) - TAG = 4; // Find all entries in database that have a specific tag + MIN_ENTRY = 3; // Find all entries in database starting from specified entry number (chronological) + SIGNER_ID = 4; // Find all entries in database that correspond to specific signature (signer) + TAG = 5; // Find all entries in database that have a specific tag // NOTE: The MAX_MESSAGES filter must remain the last filter type // This is because the condition it specifies in an SQL statement must come last in the statement // Keeping it last here allows for easily sorting the filters and keeping the code general - MAX_MESSAGES = 5; // Return at most some specified number of messages + MAX_MESSAGES = 6; // Return at most some specified number of messages } message MessageFilter { @@ -98,6 +99,11 @@ message BatchData { bytes data = 1; } +// List of BatchData; Only used for testing +message BatchDataList { + repeated BatchData data = 1; +} + // These messages comprise a batch message message BatchMessage { bytes signerId = 1; // Unique signer identifier From aeb7c13436af18dde9455c82c40221ca0f6ba442 Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Mon, 22 Feb 2016 08:04:01 +0200 Subject: [PATCH 10/12] Made read operations stream the results. Removed dependency on large Protobufs (BulletinBoardMessageList and BatchDataList). Partial implementation of Sync Query. Current version supports only H2 and MySQL (no SQLite support). --- .../bulletinboard/BulletinClientWorker.java | 2 +- .../bulletinboard/MultiServerWorker.java | 4 +- .../SimpleBulletinBoardClient.java | 2 +- .../SingleServerBulletinBoardClient.java | 56 ++-- .../bulletinboard/SingleServerWorker.java | 2 +- .../ThreadedBulletinBoardClient.java | 9 +- .../MultiServerGenericPostWorker.java | 2 - .../MultiServerGenericReadWorker.java | 4 +- .../MultiServerGetRedundancyWorker.java | 6 +- .../SingleServerGenericPostWorker.java | 4 +- .../SingleServerGetRedundancyWorker.java | 4 +- ...dedBulletinBoardClientIntegrationTest.java | 15 +- bulletin-board-server/build.gradle | 4 + .../sqlserver/BulletinBoardSQLServer.java | 265 ++++++++++++++---- .../sqlserver/H2QueryProvider.java | 18 +- .../sqlserver/MySQLQueryProvider.java | 27 +- .../mappers/BatchDataCallbackHandler.java | 32 +++ .../mappers/MessageCallbackHandler.java | 80 ++++++ .../sqlserver/mappers/MessageStubMapper.java | 31 ++ .../webapp/BulletinBoardWebApp.java | 94 ++++++- ...BulletinBoardSQLServerIntegrationTest.java | 38 ++- .../GenericBulletinBoardServerTest.java | 138 ++++++--- meerkat-common/build.gradle | 1 + .../bulletinboard/BulletinBoardConstants.java | 1 + .../bulletinboard/BulletinBoardServer.java | 19 +- .../meerkat/bulletinboard/CompleteBatch.java | 28 ++ .../java/meerkat/comm/MessageInputStream.java | 64 +++++ .../meerkat/comm/MessageOutputStream.java | 24 ++ .../src/main/java/meerkat/comm/Timestamp.java | 7 - .../util/BulletinBoardMessageGenerator.java | 98 +++++++ .../java/meerkat/util/BulletinBoardUtils.java | 42 +++ .../meerkat/util/TimeStampComparator.java | 30 ++ .../main/proto/meerkat/BulletinBoardAPI.proto | 51 +++- .../java/meerkat/comm/MessageStreamTest.java | 98 +++++++ 34 files changed, 1106 insertions(+), 194 deletions(-) create mode 100644 bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataCallbackHandler.java create mode 100644 bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageCallbackHandler.java create mode 100644 bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubMapper.java create mode 100644 meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java create mode 100644 meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java delete mode 100644 meerkat-common/src/main/java/meerkat/comm/Timestamp.java create mode 100644 meerkat-common/src/main/java/meerkat/util/BulletinBoardMessageGenerator.java create mode 100644 meerkat-common/src/main/java/meerkat/util/TimeStampComparator.java create mode 100644 meerkat-common/src/test/java/meerkat/comm/MessageStreamTest.java diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java index dba596b..1a4b62f 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java @@ -8,7 +8,7 @@ package meerkat.bulletinboard; */ public abstract class BulletinClientWorker { - protected IN payload; // Payload of the job + protected final IN payload; // Payload of the job private int maxRetry; // Number of retries for this job; set to -1 for infinite retries diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java index 727a922..7347f47 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java @@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicInteger; */ public abstract class MultiServerWorker extends BulletinClientWorker implements Runnable, FutureCallback{ - private List clients; + private final List clients; protected AtomicInteger minServers; // The minimal number of servers the job must be successful on for the job to be completed @@ -26,7 +26,7 @@ public abstract class MultiServerWorker extends BulletinClientWorker futureCallback; + private final FutureCallback futureCallback; /** * Constructor diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java index 8cac04d..633e495 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java @@ -125,7 +125,7 @@ public class SimpleBulletinBoardClient implements BulletinBoardClient{ * If at the operation is successful for some DB: return the results and stop iterating * If no operation is successful: return null (NOT blank list) * @param filterList return only messages that match the filters (null means no filtering). - * @return + * @return the list of Bulletin Board messages that are returned from a server */ @Override public List readMessages(MessageFilterList filterList) { diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java index e6070a5..ea0e93d 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java @@ -6,10 +6,12 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import meerkat.bulletinboard.workers.singleserver.*; +import meerkat.comm.CommunicationException; import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.protobuf.Voting.BulletinBoardClientParams; import meerkat.util.BulletinBoardUtils; +import javax.ws.rs.NotFoundException; import java.util.Arrays; import java.util.Iterator; import java.util.LinkedList; @@ -31,15 +33,15 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i private final int MAX_RETRIES = 11; - protected ListeningScheduledExecutorService executorService; + private ListeningScheduledExecutorService executorService; protected BatchDigest batchDigest; private long lastServerErrorTime; - protected final long failDelayInMilliseconds; + private final long failDelayInMilliseconds; - protected final long subscriptionIntervalInMilliseconds; + private final long subscriptionIntervalInMilliseconds; /** * Notify the client that a job has failed @@ -86,8 +88,8 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i */ class RetryCallback implements FutureCallback { - private SingleServerWorker worker; - private FutureCallback futureCallback; + private final SingleServerWorker worker; + private final FutureCallback futureCallback; public RetryCallback(SingleServerWorker worker, FutureCallback futureCallback) { this.worker = worker; @@ -128,7 +130,8 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i */ class PostBatchDataListCallback implements FutureCallback { - private FutureCallback callback; + private final FutureCallback callback; + private AtomicInteger batchDataRemaining; private AtomicBoolean aggregatedResult; @@ -168,7 +171,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i */ class CompleteBatchReadCallback { - private FutureCallback callback; + private final FutureCallback callback; private List batchDataList; private BulletinBoardMessage batchMessage; @@ -176,7 +179,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i private AtomicInteger remainingQueries; private AtomicBoolean failed; - public CompleteBatchReadCallback(FutureCallback callback) { + public CompleteBatchReadCallback(FutureCallback callback) { this.callback = callback; @@ -193,11 +196,16 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i if (remainingQueries.decrementAndGet() == 0){ + String batchIdStr = BulletinBoardUtils.findTagWithPrefix(batchMessage, BulletinBoardConstants.BATCH_ID_TAG_PREFIX); + + if (batchIdStr == null){ + callback.onFailure(new CommunicationException("Server returned invalid message with no Batch ID tag")); + } + BeginBatchMessage beginBatchMessage = BeginBatchMessage.newBuilder() .setSignerId(batchMessage.getSig(0).getSignerId()) - .setBatchId(Integer.parseInt( - BulletinBoardUtils.findTagWithPrefix(batchMessage, BulletinBoardConstants.BATCH_ID_TAG_PREFIX))) + .setBatchId(Integer.parseInt(batchIdStr)) .addAllTag(BulletinBoardUtils.removePrefixTags(batchMessage, Arrays.asList(prefixes))) .build(); callback.onSuccess(new CompleteBatch(beginBatchMessage, batchDataList, batchMessage.getSig(0))); @@ -267,7 +275,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i class SubscriptionCallback implements FutureCallback> { private SingleServerReadMessagesWorker worker; - private MessageHandler messageHandler; + private final MessageHandler messageHandler; private MessageFilterList.Builder filterBuilder; @@ -339,7 +347,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i // Remove all but first DB address String dbAddress = meerkatDBs.get(0); - meerkatDBs = new LinkedList(); + meerkatDBs = new LinkedList<>(); meerkatDBs.add(dbAddress); } @@ -351,7 +359,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i SingleServerPostMessageWorker worker = new SingleServerPostMessageWorker(meerkatDBs.get(0), msg, MAX_RETRIES); // Submit worker and create callback - scheduleWorker(worker, new RetryCallback(worker, callback)); + scheduleWorker(worker, new RetryCallback<>(worker, callback)); // Calculate the correct message ID and return it batchDigest.reset(); @@ -362,8 +370,8 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i private class PostBatchDataCallback implements FutureCallback { - private CompleteBatch completeBatch; - FutureCallback callback; + private final CompleteBatch completeBatch; + private final FutureCallback callback; public PostBatchDataCallback(CompleteBatch completeBatch, FutureCallback callback) { this.completeBatch = completeBatch; @@ -391,8 +399,8 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i private class BeginBatchCallback implements FutureCallback { - private CompleteBatch completeBatch; - FutureCallback callback; + private final CompleteBatch completeBatch; + private final FutureCallback callback; public BeginBatchCallback(CompleteBatch completeBatch, FutureCallback callback) { this.completeBatch = completeBatch; @@ -438,7 +446,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i new SingleServerBeginBatchWorker(meerkatDBs.get(0), beginBatchMessage, MAX_RETRIES); // Submit worker and create callback - scheduleWorker(worker, new RetryCallback(worker, callback)); + scheduleWorker(worker, new RetryCallback<>(worker, callback)); } @@ -464,7 +472,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i new SingleServerPostBatchWorker(meerkatDBs.get(0), builder.build(), MAX_RETRIES); // Create worker with redundancy 1 and MAX_RETRIES retries - scheduleWorker(worker, new RetryCallback(worker, listCallback)); + scheduleWorker(worker, new RetryCallback<>(worker, listCallback)); // Increment position in batch startPosition++; @@ -502,7 +510,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i new SingleServerCloseBatchWorker(meerkatDBs.get(0), closeBatchMessage, MAX_RETRIES); // Submit worker and create callback - scheduleWorker(worker, new RetryCallback(worker, callback)); + scheduleWorker(worker, new RetryCallback<>(worker, callback)); } @@ -513,7 +521,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i SingleServerGetRedundancyWorker worker = new SingleServerGetRedundancyWorker(meerkatDBs.get(0), id, 1); // Submit job and create callback - scheduleWorker(worker, new RetryCallback(worker, callback)); + scheduleWorker(worker, new RetryCallback<>(worker, callback)); } @@ -524,7 +532,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterList, 1); // Submit job and create callback - scheduleWorker(worker, new RetryCallback(worker, callback)); + scheduleWorker(worker, new RetryCallback<>(worker, callback)); } @@ -557,8 +565,8 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i CompleteBatchReadCallback completeBatchReadCallback = new CompleteBatchReadCallback(callback); // Submit jobs with wrapped callbacks - scheduleWorker(messageWorker, new RetryCallback(messageWorker, completeBatchReadCallback.asBulletinBoardMessageListFutureCallback())); - scheduleWorker(batchWorker, new RetryCallback(batchWorker, completeBatchReadCallback.asBatchDataListFutureCallback())); + scheduleWorker(messageWorker, new RetryCallback<>(messageWorker, completeBatchReadCallback.asBulletinBoardMessageListFutureCallback())); + scheduleWorker(batchWorker, new RetryCallback<>(batchWorker, completeBatchReadCallback.asBatchDataListFutureCallback())); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java index 82ca886..ecebc12 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java @@ -25,7 +25,7 @@ public abstract class SingleServerWorker extends BulletinClientWorker clients; + private List clients; - BatchDigest batchDigest; + private BatchDigest batchDigest; private final static int POST_MESSAGE_RETRY_NUM = 3; private final static int READ_MESSAGES_RETRY_NUM = 1; @@ -59,7 +59,7 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple executorService = Executors.newFixedThreadPool(JOBS_THREAD_NUM); - clients = new ArrayList(clientParams.getBulletinBoardAddressCount()); + clients = new ArrayList<>(clientParams.getBulletinBoardAddressCount()); for (String address : clientParams.getBulletinBoardAddressList()){ SingleServerBulletinBoardClient client = @@ -80,7 +80,6 @@ public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient imple * Retry failed DBs * @param msg is the message, * @return the message ID for later retrieval - * @throws CommunicationException */ @Override public MessageID postMessage(BulletinBoardMessage msg, FutureCallback callback){ diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java index 4ff96b1..8172e14 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java @@ -31,8 +31,6 @@ public abstract class MultiServerGenericPostWorker extends MultiServerWorker< * It accesses the servers one by one and tries to post the payload to each in turn * The method will only iterate once through the server list * Successful post to a server results in removing the server from the list - * @return The original job, but with a modified server list - * @throws CommunicationException */ public void run() { diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java index 68fc020..88b4ac1 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java @@ -14,7 +14,7 @@ import java.util.List; */ public abstract class MultiServerGenericReadWorker extends MultiServerWorker{ - protected Iterator clientIterator; + private final Iterator clientIterator; public MultiServerGenericReadWorker(List clients, int minServers, IN payload, int maxRetry, @@ -32,8 +32,6 @@ public abstract class MultiServerGenericReadWorker extends MultiServerW * This method carries out the actual communication with the servers via HTTP Post * It accesses the servers in a random order until one answers it * Successful retrieval from any server terminates the method and returns the received values; The list is not changed - * @return The original job and the list of messages found in the first server that answered the query - * @throws CommunicationException */ public void run(){ diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java index 517dbdf..748916b 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java @@ -33,8 +33,6 @@ public class MultiServerGetRedundancyWorker extends MultiServerWorker= getClientNumber()){ - succeed(new Float(((float) serversContainingMessage.get()) / ((float) getClientNumber()) )); + succeed(((float) serversContainingMessage.get()) / ((float) getClientNumber())); } } @Override public void onFailure(Throwable t) { - onSuccess(new Float(0.0)); + onSuccess(0.0f); } } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java index c56af05..621a828 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java @@ -19,7 +19,7 @@ import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER */ public class SingleServerGenericPostWorker extends SingleServerWorker { - private String subPath; + private final String subPath; public SingleServerGenericPostWorker(String serverAddress, String subPath, T payload, int maxRetry) { super(serverAddress, payload, maxRetry); @@ -37,7 +37,7 @@ public class SingleServerGenericPostWorker extends SingleServerWorker 0){ // Message exists in the server - return new Float(1.0); + return 1.0f; } else { // Message does not exist in the server - return new Float(0.0); + return 0.0f; } } catch (ProcessingException | IllegalStateException e) { diff --git a/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java b/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java index c266086..dfccebc 100644 --- a/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java +++ b/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java @@ -43,8 +43,8 @@ public class ThreadedBulletinBoardClientIntegrationTest { private static String KEYFILE_PASSWORD1 = "secret"; private static String KEYFILE_PASSWORD3 = "shh"; - public static String CERT1_PEM_EXAMPLE = "/certs/enduser-certs/user1.crt"; - public static String CERT3_PEM_EXAMPLE = "/certs/enduser-certs/user3.crt"; + private static String CERT1_PEM_EXAMPLE = "/certs/enduser-certs/user1.crt"; + private static String CERT3_PEM_EXAMPLE = "/certs/enduser-certs/user3.crt"; // Server data @@ -81,7 +81,7 @@ public class ThreadedBulletinBoardClientIntegrationTest { InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE); char[] password = KEYFILE_PASSWORD1.toCharArray(); - KeyStore.Builder keyStoreBuilder = null; + KeyStore.Builder keyStoreBuilder; try { keyStoreBuilder = signers[0].getPKCS12KeyStoreBuilder(keyStream, password); @@ -304,11 +304,11 @@ public class ThreadedBulletinBoardClientIntegrationTest { random = new Random(0); // We use insecure randomness in tests for repeatability - List testDB = new LinkedList(); + List testDB = new LinkedList<>(); testDB.add(BASE_URL); bulletinBoardClient.init(BulletinBoardClientParams.newBuilder() - .addBulletinBoardAddress("http://localhost:8081") + .addAllBulletinBoardAddress(testDB) .setMinRedundancy((float) 1.0) .build()); @@ -344,7 +344,6 @@ public class ThreadedBulletinBoardClientIntegrationTest { byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4}; byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14}; byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24}; - byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0}; BulletinBoardMessage msg; @@ -353,8 +352,6 @@ public class ThreadedBulletinBoardClientIntegrationTest { MessageID messageID; - Comparator msgComparator = new BulletinBoardMessageComparator(); - msg = BulletinBoardMessage.newBuilder() .setMsg(UnsignedBulletinBoardMessage.newBuilder() .addTag("Signature") @@ -398,7 +395,7 @@ public class ThreadedBulletinBoardClientIntegrationTest { ) .build(); - msgList = new LinkedList(); + msgList = new LinkedList<>(); msgList.add(msg); readCallback = new ReadCallback(msgList); diff --git a/bulletin-board-server/build.gradle b/bulletin-board-server/build.gradle index 59045a2..8d824e7 100644 --- a/bulletin-board-server/build.gradle +++ b/bulletin-board-server/build.gradle @@ -101,6 +101,10 @@ task dbTest(type: Test) { outputs.upToDateWhen { false } } +task manualIntegration(type: Test) { + include '**/*IntegrationTest*' +} + task integrationTest(type: Test) { include '**/*IntegrationTest*' // debug = true diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java index a6313b6..b4e2949 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java @@ -1,13 +1,9 @@ package meerkat.bulletinboard.sqlserver; -import java.security.InvalidKeyException; -import java.security.SignatureException; -import java.security.cert.CertificateException; import java.sql.*; import java.util.*; -import com.google.protobuf.ByteString; -import com.google.protobuf.ProtocolStringList; +import com.google.protobuf.*; import meerkat.bulletinboard.*; import meerkat.bulletinboard.sqlserver.mappers.*; @@ -15,6 +11,7 @@ import static meerkat.bulletinboard.BulletinBoardConstants.*; import meerkat.comm.CommunicationException; +import meerkat.comm.MessageOutputStream; import meerkat.crypto.concrete.ECDSASignature; import meerkat.crypto.concrete.SHA256Digest; @@ -27,6 +24,8 @@ import static meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryPro import javax.sql.DataSource; +import meerkat.util.BulletinBoardUtils; +import meerkat.util.TimestampComparator; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.jdbc.support.GeneratedKeyHolder; @@ -62,8 +61,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ ), INSERT_MSG( - new String[] {"MsgId","Msg"}, - new int[] {Types.BLOB, Types.BLOB} + new String[] {"MsgId","TimeStamp","Msg"}, + new int[] {Types.BLOB, Types.TIMESTAMP, Types.BLOB} ), INSERT_NEW_TAG( @@ -91,6 +90,21 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ new int[] {} ), + COUNT_MESSAGES( + new String[] {}, + new int[] {} + ), + + GET_MESSAGE_STUBS( + new String[] {}, + new int[] {} + ), + + GET_LAST_MESSAGE_ENTRY( + new String[] {}, + new int[] {} + ), + GET_BATCH_MESSAGE_ENTRY( new String[] {"SignerId", "BatchId"}, new int[] {Types.BLOB, Types.INTEGER} @@ -161,7 +175,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ MSG_ID("MsgId", Types.BLOB), SIGNER_ID("SignerId", Types.BLOB), TAG("Tag", Types.VARCHAR), - LIMIT("Limit", Types.INTEGER); + LIMIT("Limit", Types.INTEGER), + TIMESTAMP("TimeStamp", Types.TIMESTAMP); private FilterTypeParam(String paramName, int paramType) { this.paramName = paramName; @@ -191,6 +206,10 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ case MAX_MESSAGES: return LIMIT; + case BEFORE_TIME: // Go through + case AFTER_TIME: + return TIMESTAMP; + default: return null; } @@ -269,6 +288,10 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ case MAX_MESSAGES: return messageFilter.getMaxMessages(); + case BEFORE_TIME: // Go through + case AFTER_TIME: + return BulletinBoardUtils.toSQLTimestamp(messageFilter.getTimestamp()); + default: // Unsupported filter type return null; } @@ -316,8 +339,6 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ */ private void createSchema() throws SQLException { - final int TIMEOUT = 20; - for (String command : sqlQueryProvider.getSchemaCreationCommands()) { jdbcTemplate.update(command,(Map) null); } @@ -423,7 +444,6 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ // Add message to table if needed and store entry number of message. - sql = sqlQueryProvider.getSQLString(QueryType.FIND_MSG_ID); Map namedParameters = new HashMap(); namedParameters.put(QueryType.FIND_MSG_ID.getParamName(0),msgID); @@ -437,7 +457,9 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } else{ sql = sqlQueryProvider.getSQLString(QueryType.INSERT_MSG); - namedParameters.put(QueryType.INSERT_MSG.getParamName(1), msg.getMsg().toByteArray()); + + namedParameters.put(QueryType.INSERT_MSG.getParamName(1), BulletinBoardUtils.toSQLTimestamp(msg.getMsg().getTimestamp())); + namedParameters.put(QueryType.INSERT_MSG.getParamName(2), msg.getMsg().toByteArray()); KeyHolder keyHolder = new GeneratedKeyHolder(); jdbcTemplate.update(sql,new MapSqlParameterSource(namedParameters),keyHolder); @@ -504,37 +526,36 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ return postMessage(msg, true); // Perform a post and check the signature for authenticity } - @Override - public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException { - BulletinBoardMessageList.Builder resultListBuilder = BulletinBoardMessageList.newBuilder(); + /** + * This is a container class for and SQL string builder and a MapSqlParameterSource to be used with it + */ + class SQLAndParameters { - // SQL length is roughly 50 characters per filter + 50 for the query itself - StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1)); + public StringBuilder sql; + public MapSqlParameterSource parameters; - MapSqlParameterSource namedParameters; - int paramNum; + public SQLAndParameters(int numOfFilters) { + sql = new StringBuilder(50 * numOfFilters); + parameters = new MapSqlParameterSource(); + } - MessageMapper messageMapper = new MessageMapper(); - SignatureMapper signatureMapper = new SignatureMapper(); + } + + SQLAndParameters getSQLFromFilters(MessageFilterList filterList) { + + SQLAndParameters result = new SQLAndParameters(filterList.getFilterCount()); List filters = new ArrayList(filterList.getFilterList()); - boolean isFirstFilter = true; - Collections.sort(filters, new FilterTypeComparator()); - // Check if Tag/Signature tables are required for filtering purposes - - sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.GET_MESSAGES)); - // Add conditions - - namedParameters = new MapSqlParameterSource(); + boolean isFirstFilter = true; if (!filters.isEmpty()) { - sqlBuilder.append(" WHERE "); + result.sql.append(" WHERE "); - for (paramNum = 0 ; paramNum < filters.size() ; paramNum++) { + for (int paramNum = 0 ; paramNum < filters.size() ; paramNum++) { MessageFilter filter = filters.get(paramNum); @@ -542,15 +563,15 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ if (isFirstFilter) { isFirstFilter = false; } else { - sqlBuilder.append(" AND "); + result.sql.append(" AND "); } } - sqlBuilder.append(sqlQueryProvider.getCondition(filter.getType(), paramNum)); + result.sql.append(sqlQueryProvider.getCondition(filter.getType(), paramNum)); FilterTypeParam filterTypeParam = FilterTypeParam.getFilterTypeParamName(filter.getType()); - namedParameters.addValue( + result.parameters.addValue( filterTypeParam.getParamName() + Integer.toString(paramNum), getParam(filter), filterTypeParam.getParamType(), @@ -560,36 +581,56 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } + return result; + + } + + + /** + * Used to retrieve just basic information about messages to allow calculation of checksum + * @param filterList is a filter list that defines which messages the client is interested in + * @return a list of Bulletin Board Messages that contain just the entry number, timestamp and message ID for each message + * The message ID is returned inside the message data field + */ + protected List readMessageStubs(MessageFilterList filterList) { + + StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1)); + + sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.GET_MESSAGE_STUBS)); + + // Get Conditions + + SQLAndParameters sqlAndParameters = getSQLFromFilters(filterList); + + sqlBuilder.append(sqlAndParameters.sql); + // Run query - List msgBuilders = - jdbcTemplate.query(sqlBuilder.toString(), namedParameters, messageMapper); + return jdbcTemplate.query(sqlBuilder.toString(), sqlAndParameters.parameters, new MessageStubMapper()); - // Compile list of messages + } - for (BulletinBoardMessage.Builder msgBuilder : msgBuilders) { - // Retrieve signatures + @Override + public void readMessages(MessageFilterList filterList, MessageOutputStream out) throws CommunicationException { - namedParameters = new MapSqlParameterSource(); - namedParameters.addValue(QueryType.GET_SIGNATURES.getParamName(0), msgBuilder.getEntryNum()); + BulletinBoardMessageList.Builder resultListBuilder = BulletinBoardMessageList.newBuilder(); - List signatures = jdbcTemplate.query( - sqlQueryProvider.getSQLString(QueryType.GET_SIGNATURES), - namedParameters, - signatureMapper); + // SQL length is roughly 50 characters per filter + 50 for the query itself + StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1)); - // Append signatures - msgBuilder.addAllSig(signatures); + // Check if Tag/Signature tables are required for filtering purposes - // Finalize message and add to message list. + sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.GET_MESSAGES)); - resultListBuilder.addMessage(msgBuilder.build()); + // Get conditions - } + SQLAndParameters sqlAndParameters = getSQLFromFilters(filterList); + sqlBuilder.append(sqlAndParameters.sql); - //Combine results and return. - return resultListBuilder.build(); + // Run query and stream the output using a MessageCallbackHandler + + jdbcTemplate.query(sqlBuilder.toString(), sqlAndParameters.parameters, new MessageCallbackHandler(jdbcTemplate, sqlQueryProvider, out)); } @@ -625,9 +666,23 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ .build()) .build(); - BulletinBoardMessageList messageList = readMessages(filterList); + // SQL length is roughly 50 characters per filter + 50 for the query itself + StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1)); - return (messageList.getMessageList().size() > 0); + // Check if Tag/Signature tables are required for filtering purposes + + sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.COUNT_MESSAGES)); + + // Get conditions + + SQLAndParameters sqlAndParameters = getSQLFromFilters(filterList); + sqlBuilder.append(sqlAndParameters.sql); + + // Run query and stream the output using a MessageCallbackHandler + + List count = jdbcTemplate.query(sqlBuilder.toString(), sqlAndParameters.parameters, new LongMapper()); + + return (count.size() > 0) && (count.get(0) > 0); } @@ -767,6 +822,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ .addTag(BATCH_TAG) .addTag(batchIdToTag(batchId)) .setData(message.getSig().getSignerId()) + .setTimestamp(message.getTimestamp()) .build()) .build(); @@ -788,7 +844,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } @Override - public BatchDataList readBatch(BatchSpecificationMessage message) throws CommunicationException, IllegalArgumentException{ + public void readBatch(BatchSpecificationMessage message, MessageOutputStream out) throws CommunicationException, IllegalArgumentException{ // Check that batch is closed if (!isBatchClosed(message.getSignerId(), message.getBatchId())) { @@ -802,9 +858,102 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),message.getBatchId()); namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),message.getStartPosition()); - return BatchDataList.newBuilder() - .addAllData(jdbcTemplate.query(sql, namedParameters, new BatchDataMapper())) - .build(); + jdbcTemplate.query(sql, namedParameters, new BatchDataCallbackHandler(out)); + + } + + /** + * Finds the entry number of the last entry in the database + * @return the entry number, or -1 if no entries are found + */ + protected long getLastMessageEntry() { + + String sql = sqlQueryProvider.getSQLString(QueryType.GET_LAST_MESSAGE_ENTRY); + + List resultList = jdbcTemplate.query(sql, new LongMapper()); + + if (resultList.size() <= 0){ + return -1; + } + + return resultList.get(0); + + } + + /** + * Searches for the latest time of sync of the DB relative to a given query and returns the metadata needed to complete the sync + * The checksum up to (and including) each given timestamp is calculated using bitwise XOR on 8-byte sized blocks of the message IDs + * @param syncQuery contains a succinct representation of states to compare to + * @return the current last entry num and latest time of sync if there is one; -1 as last entry and empty timestamp otherwise + * @throws CommunicationException + */ + @Override + public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException { + + if (syncQuery == null){ + return SyncQueryResponse.newBuilder() + .setLastEntryNum(-1) + .setLastTimeOfSync(com.google.protobuf.Timestamp.getDefaultInstance()) + .build(); + } + + com.google.protobuf.Timestamp lastTimeOfSync = null; + + TimestampComparator timestampComparator = new TimestampComparator(); + + long lastEntryNum = getLastMessageEntry(); + + long checksum = 0; + + Iterator queryIterator = syncQuery.getQueryList().iterator(); + + SingleSyncQuery currentQuery = queryIterator.next(); + + List messageStubs = readMessageStubs(syncQuery.getFilterList()); + + for (BulletinBoardMessage message : messageStubs){ + + // Check for end of current query + if (timestampComparator.compare(message.getMsg().getTimestamp(), currentQuery.getTimeOfSync()) > 0){ + + if (checksum == currentQuery.getChecksum()){ + lastTimeOfSync = currentQuery.getTimeOfSync(); + } else { + break; + } + + if (queryIterator.hasNext()){ + currentQuery = queryIterator.next(); + } else{ + break; + } + + } + + // Advance checksum + + ByteString messageID = message.getMsg().getData(); + + checksum &= messageID.byteAt(0) & messageID.byteAt(1) & messageID.byteAt(2) & messageID.byteAt(3); + + } + + if (checksum == currentQuery.getChecksum()){ + lastTimeOfSync = currentQuery.getTimeOfSync(); + } + + if (lastTimeOfSync == null){ + return SyncQueryResponse.newBuilder() + .setLastEntryNum(-1) + .setLastTimeOfSync(com.google.protobuf.Timestamp.getDefaultInstance()) + .build(); + } else{ + return SyncQueryResponse.newBuilder() + .setLastEntryNum(lastEntryNum) + .setLastTimeOfSync(lastTimeOfSync) + .build(); + } + } @Override diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java index 659d9c3..c390048 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java @@ -51,6 +51,12 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider case GET_MESSAGES: return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"; + case COUNT_MESSAGES: + return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable"; + + case GET_MESSAGE_STUBS: + return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable"; + case GET_SIGNATURES: return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum"; @@ -61,6 +67,9 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider return "INSERT INTO TagTable(Tag) SELECT DISTINCT :Tag AS NewTag FROM UtilityTable WHERE" + " NOT EXISTS (SELECT 1 FROM TagTable AS SubTable WHERE SubTable.Tag = :Tag)"; + case GET_LAST_MESSAGE_ENTRY: + return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable"; + case GET_BATCH_MESSAGE_ENTRY: return MessageFormat.format( "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable" @@ -147,6 +156,13 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider return "EXISTS (SELECT 1 FROM TagTable" + " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId" + " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)"; + + case BEFORE_TIME: + return "MsgTable.ExactTime <= :TimeStamp"; + + case AFTER_TIME: + return "MsgTable.ExactTime >= :TimeStamp"; + default: throw new IllegalArgumentException("Cannot serve a filter of type " + filterType); } @@ -190,7 +206,7 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider public List getSchemaCreationCommands() { List list = new LinkedList(); - list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB UNIQUE, Msg BLOB)"); + list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB UNIQUE, ExactTime TIMESTAMP, Msg BLOB)"); list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INT NOT NULL AUTO_INCREMENT PRIMARY KEY, Tag VARCHAR(50) UNIQUE)"); diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java index e3bdef0..f99114e 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java @@ -62,6 +62,12 @@ public class MySQLQueryProvider implements SQLQueryProvider { case GET_MESSAGES: return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"; + case COUNT_MESSAGES: + return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable"; + + case GET_MESSAGE_STUBS: + return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable"; + case GET_SIGNATURES: return MessageFormat.format( "SELECT Signature FROM SignatureTable WHERE EntryNum = :{0}", @@ -69,15 +75,19 @@ public class MySQLQueryProvider implements SQLQueryProvider { case INSERT_MSG: return MessageFormat.format( - "INSERT INTO MsgTable (MsgId, Msg) VALUES(:{0}, :{1})", + "INSERT INTO MsgTable (MsgId, ExactTime, Msg) VALUES(:{0}, :{1}, :{2})", QueryType.INSERT_MSG.getParamName(0), - QueryType.INSERT_MSG.getParamName(1)); + QueryType.INSERT_MSG.getParamName(1), + QueryType.INSERT_MSG.getParamName(2)); case INSERT_NEW_TAG: return MessageFormat.format( "INSERT IGNORE INTO TagTable(Tag) VALUES (:{0})", QueryType.INSERT_NEW_TAG.getParamName(0)); + case GET_LAST_MESSAGE_ENTRY: + return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable"; + case GET_BATCH_MESSAGE_ENTRY: return MessageFormat.format( "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable" @@ -164,6 +174,13 @@ public class MySQLQueryProvider implements SQLQueryProvider { return "EXISTS (SELECT 1 FROM TagTable" + " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId" + " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)"; + + case BEFORE_TIME: + return "MsgTable.ExactTime <= :TimeStamp"; + + case AFTER_TIME: + return "MsgTable.ExactTime >= :TimeStamp"; + default: throw new IllegalArgumentException("Cannot serve a filter of type " + filterType); } @@ -187,6 +204,10 @@ public class MySQLQueryProvider implements SQLQueryProvider { case TAG: return "VARCHAR"; + case AFTER_TIME: // Go through + case BEFORE_TIME: + return "TIMESTAMP"; + default: throw new IllegalArgumentException("Cannot serve a filter of type " + filterType); } @@ -212,7 +233,7 @@ public class MySQLQueryProvider implements SQLQueryProvider { List list = new LinkedList(); list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY," - + " MsgId TINYBLOB, Msg BLOB, UNIQUE(MsgId(50)))"); + + " MsgId TINYBLOB, ExactTime TIMESTAMP, Msg BLOB, UNIQUE(MsgId(50)))"); list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INT NOT NULL AUTO_INCREMENT PRIMARY KEY, Tag VARCHAR(50), UNIQUE(Tag))"); diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataCallbackHandler.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataCallbackHandler.java new file mode 100644 index 0000000..9ad0dc7 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataCallbackHandler.java @@ -0,0 +1,32 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import com.google.protobuf.InvalidProtocolBufferException; +import meerkat.comm.MessageOutputStream; +import meerkat.protobuf.BulletinBoardAPI.BatchData; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.jdbc.core.RowMapper; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by Arbel Deutsch Peled on 19-Dec-15. + */ +public class BatchDataCallbackHandler implements RowCallbackHandler { + + private final MessageOutputStream out; + + public BatchDataCallbackHandler(MessageOutputStream out) { + this.out = out; + } + + @Override + public void processRow(ResultSet rs) throws SQLException { + try { + out.writeMessage(BatchData.parseFrom(rs.getBytes(1))); + } catch (IOException e) { + //TODO: Log + } + } +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageCallbackHandler.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageCallbackHandler.java new file mode 100644 index 0000000..bdba241 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageCallbackHandler.java @@ -0,0 +1,80 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import com.google.protobuf.InvalidProtocolBufferException; +import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.*; +import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider.*; +import meerkat.comm.MessageOutputStream; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Crypto; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + */ +public class MessageCallbackHandler implements RowCallbackHandler { + + NamedParameterJdbcTemplate jdbcTemplate; + SQLQueryProvider sqlQueryProvider; + MessageOutputStream out; + + public MessageCallbackHandler(NamedParameterJdbcTemplate jdbcTemplate, SQLQueryProvider sqlQueryProvider, MessageOutputStream out) { + + this.jdbcTemplate = jdbcTemplate; + this.sqlQueryProvider = sqlQueryProvider; + this.out = out; + + } + + @Override + public void processRow(ResultSet rs) throws SQLException { + + BulletinBoardMessage.Builder result; + + try { + + result = BulletinBoardMessage.newBuilder() + .setEntryNum(rs.getLong(1)) + .setMsg(UnsignedBulletinBoardMessage.parseFrom(rs.getBytes(2))); + + + } catch (InvalidProtocolBufferException e) { + //TODO: log + return; + } + + // Retrieve signatures + + MapSqlParameterSource sqlParameterSource = new MapSqlParameterSource(); + sqlParameterSource.addValue(QueryType.GET_SIGNATURES.getParamName(0), result.getEntryNum()); + + List signatures = jdbcTemplate.query( + sqlQueryProvider.getSQLString(QueryType.GET_SIGNATURES), + sqlParameterSource, + new SignatureMapper()); + + // Append signatures + result.addAllSig(signatures); + + // Finalize message and add to message list. + + try { + + out.writeMessage(result.build()); + + } catch (IOException e) { + + //TODO: log + e.printStackTrace(); + + } + + } + +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubMapper.java new file mode 100644 index 0000000..1f9c459 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubMapper.java @@ -0,0 +1,31 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; +import meerkat.protobuf.BulletinBoardAPI.UnsignedBulletinBoardMessage; +import meerkat.util.BulletinBoardUtils; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by Arbel Deutsch Peled on 11-Dec-15. + */ +public class MessageStubMapper implements RowMapper { + + @Override + public BulletinBoardMessage mapRow(ResultSet rs, int rowNum) throws SQLException { + + return BulletinBoardMessage.newBuilder() + .setEntryNum(rs.getLong(1)) + .setMsg(UnsignedBulletinBoardMessage.newBuilder() + .setData(ByteString.copyFrom(rs.getBytes(2))) + .setTimestamp(BulletinBoardUtils.toTimestampProto(rs.getTimestamp(3))) + .build()) + .build(); + + } + +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java index 766af19..2e4782f 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java @@ -3,13 +3,10 @@ package meerkat.bulletinboard.webapp; import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.StreamingOutput; import meerkat.bulletinboard.BulletinBoardServer; import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer; @@ -17,12 +14,19 @@ import meerkat.bulletinboard.sqlserver.H2QueryProvider; import meerkat.bulletinboard.sqlserver.MySQLQueryProvider; import meerkat.bulletinboard.sqlserver.SQLiteQueryProvider; import meerkat.comm.CommunicationException; +import meerkat.comm.MessageOutputStream; +import meerkat.protobuf.BulletinBoardAPI; import meerkat.protobuf.BulletinBoardAPI.*; import static meerkat.bulletinboard.BulletinBoardConstants.*; import static meerkat.rest.Constants.*; +import java.io.IOException; +import java.io.OutputStream; import java.util.List; +/** + * An implementation of the BulletinBoardServer which functions as a WebApp + */ @Path(BULLETIN_BOARD_SERVER_PATH) public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextListener{ @@ -88,15 +92,39 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL init(); return bulletinBoard.postMessage(msg); } - + + @Override + public void readMessages(MessageFilterList filterList, MessageOutputStream out) throws CommunicationException { + init(); + bulletinBoard.readMessages(filterList, out); + } + + @Path(READ_MESSAGES_PATH) @POST @Consumes(MEDIATYPE_PROTOBUF) - @Produces(MEDIATYPE_PROTOBUF) - @Override - public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException { - init(); - return bulletinBoard.readMessages(filterList); + /** + * Wrapper for the readMessages method which streams the output into the response + */ + public StreamingOutput readMessages(final MessageFilterList filterList) { + + return new StreamingOutput() { + + @Override + public void write(OutputStream output) throws IOException, WebApplicationException { + MessageOutputStream out = new MessageOutputStream<>(output); + + try { + init(); + bulletinBoard.readMessages(filterList, out); + } catch (CommunicationException e) { + //TODO: Log + out.writeMessage(null); + } + } + + }; + } @Path(BEGIN_BATCH_PATH) @@ -144,15 +172,53 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL } } + + @Override + public void readBatch(BatchSpecificationMessage message, MessageOutputStream out) { + try { + init(); + bulletinBoard.readBatch(message, out); + } catch (CommunicationException | IllegalArgumentException e) { + System.err.println(e.getMessage()); + } + } + @Path(READ_BATCH_PATH) @POST @Consumes(MEDIATYPE_PROTOBUF) + /** + * Wrapper for the readBatch method which streams the output into the response + */ + public StreamingOutput readBatch(final BatchSpecificationMessage message) { + + return new StreamingOutput() { + + @Override + public void write(OutputStream output) throws IOException, WebApplicationException { + MessageOutputStream out = new MessageOutputStream<>(output); + + try { + init(); + bulletinBoard.readBatch(message, out); + } catch (CommunicationException e) { + //TODO: Log + out.writeMessage(null); + } + } + + }; + + } + + @Path(SYNC_QUERY_PATH) + @POST + @Consumes(MEDIATYPE_PROTOBUF) @Produces(MEDIATYPE_PROTOBUF) @Override - public BatchDataList readBatch(BatchSpecificationMessage message) { - try { + public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException { + try{ init(); - return bulletinBoard.readBatch(message); + return bulletinBoard.querySync(syncQuery); } catch (CommunicationException | IllegalArgumentException e) { System.err.println(e.getMessage()); return null; diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java index 4b8b586..744a086 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java @@ -4,6 +4,8 @@ package meerkat.bulletinboard; import com.google.protobuf.ByteString; import com.google.protobuf.TextFormat; +import com.google.protobuf.Timestamp; +import meerkat.comm.MessageInputStream; import meerkat.protobuf.Crypto.*; import meerkat.protobuf.BulletinBoardAPI.*; import static meerkat.bulletinboard.BulletinBoardConstants.*; @@ -18,7 +20,10 @@ import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.io.InputStream; +import java.util.List; public class BulletinBoardSQLServerIntegrationTest { @@ -44,6 +49,16 @@ public class BulletinBoardSQLServerIntegrationTest { byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24}; byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0}; + Timestamp t1 = Timestamp.newBuilder() + .setSeconds(8276482) + .setNanos(4314) + .build(); + + Timestamp t2 = Timestamp.newBuilder() + .setSeconds(987591) + .setNanos(1513) + .build(); + WebTarget webTarget; Response response; BoolMsg bool; @@ -51,7 +66,7 @@ public class BulletinBoardSQLServerIntegrationTest { BulletinBoardMessage msg; MessageFilterList filterList; - BulletinBoardMessageList msgList; + List msgList; // Test writing mechanism @@ -64,6 +79,7 @@ public class BulletinBoardSQLServerIntegrationTest { .addTag("Signature") .addTag("Trustee") .setData(ByteString.copyFrom(b1)) + .setTimestamp(t1) .build()) .addSig(Signature.newBuilder() .setType(SignatureType.DSA) @@ -87,6 +103,7 @@ public class BulletinBoardSQLServerIntegrationTest { .addTag("Vote") .addTag("Trustee") .setData(ByteString.copyFrom(b4)) + .setTimestamp(t2) .build()) .addSig(Signature.newBuilder() .setType(SignatureType.ECDSA) @@ -113,13 +130,20 @@ public class BulletinBoardSQLServerIntegrationTest { ) .build(); - response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF)); - System.err.println(response); - msgList = response.readEntity(BulletinBoardMessageList.class); - System.err.println("List size: " + msgList.getMessageCount()); + InputStream in = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF), InputStream.class); + + MessageInputStream inputStream = + MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BulletinBoardMessage.class); + + msgList = inputStream.asList(); + System.err.println("List size: " + msgList.size()); System.err.println("This is the list:"); - System.err.println(TextFormat.printToString(msgList)); - assert msgList.getMessageCount() == 1; + + for (BulletinBoardMessage message : msgList) { + System.err.println(TextFormat.printToString(message)); + } + + assert msgList.size() == 1; } } diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java index 86a32de..e33c739 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java @@ -1,9 +1,12 @@ package meerkat.bulletinboard; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; +import java.lang.reflect.InvocationTargetException; import java.math.BigInteger; import java.security.InvalidKeyException; import java.security.KeyStore; @@ -16,17 +19,23 @@ import java.util.*; import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; import meerkat.comm.CommunicationException; +import meerkat.comm.MessageInputStream; +import meerkat.comm.MessageOutputStream; +import meerkat.comm.MessageInputStream.MessageInputStreamFactory; import meerkat.crypto.concrete.ECDSASignature; import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.util.BulletinBoardUtils; import static org.junit.Assert.*; import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.MatcherAssert.assertThat; public class GenericBulletinBoardServerTest { protected BulletinBoardServer bulletinBoardServer; - private GenericBatchDigitalSignature signers[]; + private GenericBatchDigitalSignature[] signers; private ByteString[] signerIDs; private Random random; @@ -172,7 +181,8 @@ public class GenericBulletinBoardServerTest { for (i = 1; i <= MESSAGE_NUM; i++) { unsignedMsgBuilder = UnsignedBulletinBoardMessage.newBuilder() - .setData(ByteString.copyFrom(data[i - 1])); + .setData(ByteString.copyFrom(data[i - 1])) + .setTimestamp(BulletinBoardUtils.toTimestampProto()); // Add tags based on bit-representation of message number. @@ -232,28 +242,39 @@ public class GenericBulletinBoardServerTest { System.err.println("Starting to test tag and signature mechanism"); long start = threadBean.getCurrentThreadCpuTime(); - List messages; + List messages = new LinkedList<>(); // Check tag mechanism - + for (int i = 0 ; i < TAG_NUM ; i++){ // Retrieve messages having tag i try { - messages = bulletinBoardServer.readMessages( - MessageFilterList.newBuilder() - .addFilter(MessageFilter.newBuilder() - .setType(FilterType.TAG) - .setTag(tags[i]) - .build() - ) - .build() - ) - .getMessageList(); + MessageFilterList filterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(tags[i]) + .build() + ) + .build(); - } catch (CommunicationException e) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + bulletinBoardServer.readMessages(filterList, new MessageOutputStream(outputStream)); + + MessageInputStream inputStream = + MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream( + outputStream.toByteArray()), + BulletinBoardMessage.class); + + messages = inputStream.asList(); + + } catch (CommunicationException | IOException e) { + fail(e.getMessage()); + return; + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { fail(e.getMessage()); return; } @@ -330,11 +351,26 @@ public class GenericBulletinBoardServerTest { ); try { - messages = bulletinBoardServer.readMessages(filterListBuilder.build()).getMessageList(); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + bulletinBoardServer.readMessages(filterListBuilder.build(), new MessageOutputStream(outputStream)); + + MessageInputStream inputStream = + MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream( + outputStream.toByteArray()), + BulletinBoardMessage.class); + + messages = inputStream.asList(); + } catch (CommunicationException e) { System.err.println("Failed retrieving multi-tag messages from DB: " + e.getMessage()); fail("Failed retrieving multi-tag messages from DB: " + e.getMessage()); return; + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | IOException e) { + System.err.println("Falied to read from stream while retrieving multi-tag messages: " + e.getMessage()); + fail("Falied to read from stream while retrieving multi-tag messages: " + e.getMessage()); + return; } expectedMsgCount /= 2; @@ -361,11 +397,26 @@ public class GenericBulletinBoardServerTest { .build()); try { - messages = bulletinBoardServer.readMessages(filterListBuilder.build()).getMessageList(); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + bulletinBoardServer.readMessages(filterListBuilder.build(), new MessageOutputStream(outputStream)); + + MessageInputStream inputStream = + MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream( + outputStream.toByteArray()), + BulletinBoardMessage.class); + + messages = inputStream.asList(); + } catch (CommunicationException e) { System.err.println("Failed retrieving multi-signature message from DB: " + e.getMessage()); fail("Failed retrieving multi-signature message from DB: " + e.getMessage()); return; + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | IOException e) { + System.err.println("Falied to read from stream while retrieving multi-signature message: " + e.getMessage()); + fail("Falied to read from stream while retrieving multi-signature message: " + e.getMessage()); + return; } assertThat(messages.size(), is(MESSAGE_NUM / 4)); @@ -388,7 +439,10 @@ public class GenericBulletinBoardServerTest { final int BATCH_ID = 100; - CompleteBatch completeBatch = new CompleteBatch(); + CompleteBatch completeBatch = new CompleteBatch(Timestamp.newBuilder() + .setSeconds(978325) + .setNanos(8097234) + .build()); BoolMsg result; // Create data @@ -429,11 +483,7 @@ public class GenericBulletinBoardServerTest { // Close batch - result = bulletinBoardServer.closeBatchMessage(CloseBatchMessage.newBuilder() - .setBatchId(BATCH_ID) - .setBatchLength(1) - .setSig(completeBatch.getSignature()) - .build()); + result = bulletinBoardServer.closeBatchMessage(completeBatch.getCloseBatchMessage()); assertThat("Was not able to close batch", result.getValue(), is(true)); @@ -457,7 +507,10 @@ public class GenericBulletinBoardServerTest { */ public void testPostBatch() throws CommunicationException, SignatureException { - CompleteBatch completeBatch = new CompleteBatch(); + CompleteBatch completeBatch = new CompleteBatch(Timestamp.newBuilder() + .setSeconds(12345) + .setNanos(1111) + .build()); int currentBatch = completeBatches.size(); BoolMsg result; @@ -523,11 +576,7 @@ public class GenericBulletinBoardServerTest { signers[0].updateContent(completeBatch); completeBatch.setSignature(signers[0].sign()); - result = bulletinBoardServer.closeBatchMessage(CloseBatchMessage.newBuilder() - .setBatchId(currentBatch) - .setBatchLength(tempBatchData.length) - .setSig(completeBatch.getSignature()) - .build()); + result = bulletinBoardServer.closeBatchMessage(completeBatch.getCloseBatchMessage()); assertThat("Could not close batch " + currentBatch, result.getValue(), is(true)); @@ -540,17 +589,32 @@ public class GenericBulletinBoardServerTest { for (CompleteBatch completeBatch : completeBatches) { - List batchDataList = - bulletinBoardServer.readBatch(BatchSpecificationMessage.newBuilder() - .setSignerId(completeBatch.getBeginBatchMessage().getSignerId()) - .setBatchId(completeBatch.getBeginBatchMessage().getBatchId()) - .setStartPosition(0) - .build()); + try { - assertThat("Non-matching batch data for batch " + completeBatch.getBeginBatchMessage().getBatchId(), - completeBatch.getBatchDataList().equals(batchDataList), is(true)); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BatchSpecificationMessage batchSpecificationMessage = + BatchSpecificationMessage.newBuilder() + .setSignerId(completeBatch.getBeginBatchMessage().getSignerId()) + .setBatchId(completeBatch.getBeginBatchMessage().getBatchId()) + .setStartPosition(0) + .build(); + bulletinBoardServer.readBatch(batchSpecificationMessage, new MessageOutputStream(outputStream)); + + MessageInputStream inputStream = + MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream( + outputStream.toByteArray()), + BatchData.class); + + List batchDataList = inputStream.asList(); + + assertThat("Non-matching batch data for batch " + completeBatch.getBeginBatchMessage().getBatchId(), + completeBatch.getBatchDataList().equals(batchDataList), is(true)); + + } catch (IOException | NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + assertThat("Error reading batch data list from input stream", false); + } } diff --git a/meerkat-common/build.gradle b/meerkat-common/build.gradle index 3783531..a9035b0 100644 --- a/meerkat-common/build.gradle +++ b/meerkat-common/build.gradle @@ -39,6 +39,7 @@ version += "${isSnapshot ? '-SNAPSHOT' : ''}" dependencies { // Logging compile 'org.slf4j:slf4j-api:1.7.7' + compile 'javax.ws.rs:javax.ws.rs-api:2.0.+' runtime 'ch.qos.logback:logback-classic:1.1.2' runtime 'ch.qos.logback:logback-core:1.1.2' diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java index 6bfc06f..66652f8 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java @@ -14,6 +14,7 @@ public interface BulletinBoardConstants { public static final String BEGIN_BATCH_PATH = "/beginbatch"; public static final String POST_BATCH_PATH = "/postbatch"; public static final String CLOSE_BATCH_PATH = "/closebatch"; + public static final String SYNC_QUERY_PATH = "/syncquery"; // Other Constants diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java index cbf06ff..0b279c2 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java @@ -1,9 +1,9 @@ package meerkat.bulletinboard; import meerkat.comm.CommunicationException; +import meerkat.comm.MessageOutputStream; import meerkat.protobuf.BulletinBoardAPI.*; -import java.util.List; /** * Created by Arbel on 07/11/15. @@ -32,10 +32,10 @@ public interface BulletinBoardServer{ /** * Read all messages posted matching the given filter * @param filterList return only messages that match the filters (empty list or null means no filtering) - * @return + * @param out is an output stream into which the matching messages are written * @throws CommunicationException on DB connection error */ - public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException; + public void readMessages(MessageFilterList filterList, MessageOutputStream out) throws CommunicationException; /** * Informs server about a new batch message @@ -71,11 +71,20 @@ public interface BulletinBoardServer{ /** * Reads a batch message from the server (starting with the supplied position) * @param message specifies the signer ID and the batch ID to read as well as an (optional) start position - * @return an ordered list of batch messages starting from the specified start position (if given) or from the beginning (if omitted) + * @param out is a stream of the ordered batch messages starting from the specified start position (if given) or from the beginning (if omitted) * @throws CommunicationException on DB connection error * @throws IllegalArgumentException if message does not specify a batch */ - public BatchDataList readBatch(BatchSpecificationMessage message) throws CommunicationException, IllegalArgumentException; + public void readBatch(BatchSpecificationMessage message, MessageOutputStream out) throws CommunicationException, IllegalArgumentException; + + + /** + * Queries the database for sync status with respect to a given sync query + * @param syncQuery contains a succinct representation of states to compare to + * @return a SyncQueryResponse object containing the representation of the most recent state the database matches + * @throws CommunicationException + */ + public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException; /** * This method closes the connection to the DB diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java index 227c7ca..14e87e7 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java @@ -1,5 +1,6 @@ package meerkat.bulletinboard; +import com.google.protobuf.Timestamp; import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.protobuf.Crypto.*; import meerkat.util.BulletinBoardMessageComparator; @@ -17,6 +18,7 @@ public class CompleteBatch { private BeginBatchMessage beginBatchMessage; private List batchDataList; private Signature signature; + private Timestamp timestamp; public CompleteBatch() { batchDataList = new LinkedList(); @@ -37,6 +39,16 @@ public class CompleteBatch { signature = newSignature; } + public CompleteBatch(BeginBatchMessage newBeginBatchMessage, List newDataList, Signature newSignature, Timestamp timestamp) { + this(newBeginBatchMessage, newDataList, newSignature); + this.timestamp = timestamp; + } + + public CompleteBatch(Timestamp timestamp) { + this(); + this.timestamp = timestamp; + } + public BeginBatchMessage getBeginBatchMessage() { return beginBatchMessage; } @@ -49,11 +61,16 @@ public class CompleteBatch { return signature; } + public Timestamp getTimestamp() { + return timestamp; + } + public CloseBatchMessage getCloseBatchMessage() { return CloseBatchMessage.newBuilder() .setBatchId(getBeginBatchMessage().getBatchId()) .setBatchLength(getBatchDataList().size()) .setSig(getSignature()) + .setTimestamp(getTimestamp()) .build(); } @@ -73,6 +90,10 @@ public class CompleteBatch { signature = newSignature; } + public void setTimestamp(Timestamp timestamp) { + this.timestamp = timestamp; + } + @Override public boolean equals(Object other) { @@ -105,6 +126,13 @@ public class CompleteBatch { result = result && signature.equals(otherBatch.getSignature()); } + if (timestamp == null) { + if (otherBatch.getTimestamp() != null) + return false; + } else { + result = result && timestamp.equals(otherBatch.getTimestamp()); + } + return result; } diff --git a/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java b/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java new file mode 100644 index 0000000..33deb3f --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java @@ -0,0 +1,64 @@ +package meerkat.comm; + +import com.google.protobuf.Message; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.util.LinkedList; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + * A input stream of Protobuf messages + */ +public class MessageInputStream{ + + private T.Builder builder; + + private InputStream in; + + MessageInputStream(InputStream in, Class type) throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException { + this.in = in; + this.builder = (T.Builder) type.getMethod("newBuilder").invoke(type); + } + + /** + * Factory class for actually creating a MessageInputStream + */ + public static class MessageInputStreamFactory { + + public static MessageInputStream createMessageInputStream(InputStream in, Class type) + throws IOException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { + + return new MessageInputStream<>(in, type); + + } + + } + + public T readMessage() throws IOException{ + + builder.clear(); + builder.mergeDelimitedFrom(in); + return (T) builder.build(); + + } + + public boolean isAvailable() throws IOException { + return (in.available() > 0); + } + + public List asList() throws IOException{ + + List list = new LinkedList<>(); + + while (isAvailable()){ + list.add(readMessage()); + } + + return list; + + } + +} diff --git a/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java b/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java new file mode 100644 index 0000000..b55bc8e --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java @@ -0,0 +1,24 @@ +package meerkat.comm; + +import com.google.protobuf.Message; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + * An output stream of Protobuf messages + */ +public class MessageOutputStream { + + private OutputStream out; + + public MessageOutputStream(OutputStream out) throws IOException { + this.out = out; + } + + public void writeMessage(T message) throws IOException { + message.writeDelimitedTo(out); + } + +} diff --git a/meerkat-common/src/main/java/meerkat/comm/Timestamp.java b/meerkat-common/src/main/java/meerkat/comm/Timestamp.java deleted file mode 100644 index 6c63854..0000000 --- a/meerkat-common/src/main/java/meerkat/comm/Timestamp.java +++ /dev/null @@ -1,7 +0,0 @@ -package meerkat.comm; - -/** - * Created by talm on 24/10/15. - */ -public class Timestamp { -} diff --git a/meerkat-common/src/main/java/meerkat/util/BulletinBoardMessageGenerator.java b/meerkat-common/src/main/java/meerkat/util/BulletinBoardMessageGenerator.java new file mode 100644 index 0000000..5ca3e0b --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/util/BulletinBoardMessageGenerator.java @@ -0,0 +1,98 @@ +package meerkat.util; + +import com.google.protobuf.ByteString; +import meerkat.crypto.DigitalSignature; +import meerkat.protobuf.BulletinBoardAPI.*; +import com.google.protobuf.Timestamp; + +import java.math.BigInteger; +import java.security.SignatureException; +import java.util.Arrays; +import java.util.Random; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + * This class contains methods used to generate random Bulletin Board Messages + */ +public class BulletinBoardMessageGenerator { + + private Random random; + + public BulletinBoardMessageGenerator(Random random) { + this.random = random; + } + + private byte randomByte(){ + return (byte) random.nextInt(); + } + + private String randomString(){ + return new BigInteger(130, random).toString(32); + } + + /** + * Generates a complete instance of a BulletinBoardMessage + * @param signers contains the (possibly multiple) credentials required to sign the message + * @param timestamp contains the time used in the message + * @param dataSize is the length of the data contained in the message + * @param tagNumber is the number of tags to generate + * @return a random, signed Bulletin Board Message containing random data and tags and the given timestamp + */ + + public BulletinBoardMessage generateRandomMessage(DigitalSignature[] signers, Timestamp timestamp, int dataSize, int tagNumber) + throws SignatureException { + + // Generate random data. + + byte[] data = new byte[dataSize]; + String[] tags = new String[tagNumber]; + + for (int i = 0; i < dataSize; i++) { + data[i] = randomByte(); + } + + for (int i = 0; i < tagNumber; i++) { + tags[i] = randomString(); + } + + UnsignedBulletinBoardMessage unsignedMessage = + UnsignedBulletinBoardMessage.newBuilder() + .setData(ByteString.copyFrom(data)) + .setTimestamp(timestamp) + .addAllTag(Arrays.asList(tags)) + .build(); + + BulletinBoardMessage.Builder messageBuilder = + BulletinBoardMessage.newBuilder() + .setMsg(unsignedMessage); + + for (int i = 0 ; i < signers.length ; i++) { + signers[i].updateContent(unsignedMessage); + messageBuilder.addSig(signers[i].sign()); + } + + return messageBuilder.build(); + + } + + /** + * Generates a complete instance of a BulletinBoardMessage + * @param signers contains the (possibly multiple) credentials required to sign the message + * @param dataSize is the length of the data contained in the message + * @param tagNumber is the number of tags to generate + * @return a random, signed Bulletin Board Message containing random data, tags and timestamp + */ + + public BulletinBoardMessage generateRandomMessage(DigitalSignature[] signers, int dataSize, int tagNumber) + throws SignatureException { + + Timestamp timestamp = Timestamp.newBuilder() + .setSeconds(random.nextLong()) + .setNanos(random.nextInt()) + .build(); + + return generateRandomMessage(signers, timestamp, dataSize, tagNumber); + + } + +} diff --git a/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java b/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java index b19bab3..d8b362c 100644 --- a/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java +++ b/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java @@ -62,4 +62,46 @@ public class BulletinBoardUtils { } + /** + * This method creates a Timestamp Protobuf from a time specification + * @param timeInMillis is the time to encode since the Epoch time in milliseconds + * @return a Timestamp Protobuf encoding of the given time + */ + public static com.google.protobuf.Timestamp toTimestampProto(long timeInMillis) { + + return com.google.protobuf.Timestamp.newBuilder() + .setSeconds(timeInMillis / 1000) + .setNanos((int) ((timeInMillis % 1000) * 1000000)) + .build(); + + } + + /** + * This method creates a Timestamp Protobuf from the current system time + * @return a Timestamp Protobuf encoding of the current system time + */ + public static com.google.protobuf.Timestamp toTimestampProto() { + + return toTimestampProto(System.currentTimeMillis()); + + } + + /** + * This method converts an SQL Timestamp object into a Protobuf Timestamp object + * @param sqlTimestamp is the SQL Timestamp + * @return an equivalent Protobuf Timestamp + */ + public static com.google.protobuf.Timestamp toTimestampProto(java.sql.Timestamp sqlTimestamp) { + return toTimestampProto(sqlTimestamp.getTime()); + } + + /** + * This method converts a Protobuf Timestamp object into an SQL Timestamp object + * @param protoTimestamp is the Protobuf Timestamp + * @return an equivalent SQL Timestamp + */ + public static java.sql.Timestamp toSQLTimestamp(com.google.protobuf.Timestamp protoTimestamp) { + return new java.sql.Timestamp(protoTimestamp.getSeconds() * 1000 + protoTimestamp.getNanos() / 1000000); + } + } diff --git a/meerkat-common/src/main/java/meerkat/util/TimeStampComparator.java b/meerkat-common/src/main/java/meerkat/util/TimeStampComparator.java new file mode 100644 index 0000000..1dc207b --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/util/TimeStampComparator.java @@ -0,0 +1,30 @@ +package meerkat.util; + +import com.google.protobuf.Timestamp; + +import java.util.Comparator; + +/** + * Created by Arbel Deutsch Peled on 20-Feb-16. + */ +public class TimestampComparator implements Comparator { + + @Override + public int compare(Timestamp o1, Timestamp o2) { + + if (o1.getSeconds() != o2.getSeconds()){ + + return o1.getSeconds() > o2.getSeconds() ? 2 : -2; + + } else if (o1.getNanos() != o2.getNanos()){ + + return o1.getNanos() > o2.getNanos() ? 1 : -1; + + } else{ + + return 0; + + } + + } +} diff --git a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto index b86debd..fd95503 100644 --- a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto +++ b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto @@ -5,6 +5,7 @@ package meerkat; option java_package = "meerkat.protobuf"; import 'meerkat/crypto.proto'; +import 'google/protobuf/timestamp.proto'; message BoolMsg { bool value = 1; @@ -21,11 +22,14 @@ message MessageID { } message UnsignedBulletinBoardMessage { - // Optional tags describing message + // Optional tags describing message; Used for message retrieval repeated string tag = 1; + // Timestamp of the message (as defined by client) + google.protobuf.Timestamp timestamp = 2; + // The actual content of the message - bytes data = 2; + bytes data = 3; } message BulletinBoardMessage { @@ -38,6 +42,7 @@ message BulletinBoardMessage { // Signature of message (and tags), excluding the entry number. repeated meerkat.Signature sig = 3; + } message BulletinBoardMessageList { @@ -53,11 +58,13 @@ enum FilterType { MIN_ENTRY = 3; // Find all entries in database starting from specified entry number (chronological) SIGNER_ID = 4; // Find all entries in database that correspond to specific signature (signer) TAG = 5; // Find all entries in database that have a specific tag + AFTER_TIME = 6; // Find all entries in database that occurred on or after a given timestamp + BEFORE_TIME = 7; // Find all entries in database that occurred on or before a given timestamp // NOTE: The MAX_MESSAGES filter must remain the last filter type // This is because the condition it specifies in an SQL statement must come last in the statement // Keeping it last here allows for easily sorting the filters and keeping the code general - MAX_MESSAGES = 6; // Return at most some specified number of messages + MAX_MESSAGES = 8; // Return at most some specified number of messages } message MessageFilter { @@ -69,6 +76,7 @@ message MessageFilter { int64 entry = 3; string tag = 4; int64 maxMessages = 5; + google.protobuf.Timestamp timestamp = 6; } } @@ -89,9 +97,10 @@ message BeginBatchMessage { // This message is used to finalize and sign a batch transfer to the Bulletin Board Server message CloseBatchMessage { - int32 batchId = 1; // Unique identifier for the batch (unique per signer) - int32 batchLength = 2; // Number of messages in the batch - meerkat.Signature sig = 3; // Signature on the (ordered) batch messages + int32 batchId = 1; // Unique identifier for the batch (unique per signer) + int32 batchLength = 2; // Number of messages in the batch + google.protobuf.Timestamp timestamp = 3; // Timestamp of the batch (as defined by client) + meerkat.Signature sig = 4; // Signature on the (ordered) batch messages } // Container for single batch message data @@ -117,4 +126,34 @@ message BatchSpecificationMessage { bytes signerId = 1; // Unique signer identifier int32 batchId = 2; // Unique identifier for the batch (unique per signer) int32 startPosition = 3; // Position in batch to start reading from +} + +// This message is used to define a single query to the server to ascertain whether or not the server is synched with the client +// up till a specified timestamp +message SingleSyncQuery { + + google.protobuf.Timestamp timeOfSync = 1; + int64 checksum = 2; + +} + +// This message defines a complete server sync query +message SyncQuery { + + MessageFilterList filterList = 1; + + repeated SingleSyncQuery query = 2; + +} + +// This message defines the server's response format to a sync query +message SyncQueryResponse { + + // Serial entry number of current last entry in database + // Set to zero (0) in case no query checksums match + int64 lastEntryNum = 1; + + // Largest value of timestamp for which the checksums match + google.protobuf.Timestamp lastTimeOfSync = 2; + } \ No newline at end of file diff --git a/meerkat-common/src/test/java/meerkat/comm/MessageStreamTest.java b/meerkat-common/src/test/java/meerkat/comm/MessageStreamTest.java new file mode 100644 index 0000000..58dc7c1 --- /dev/null +++ b/meerkat-common/src/test/java/meerkat/comm/MessageStreamTest.java @@ -0,0 +1,98 @@ +package meerkat.comm; + +import com.google.protobuf.*; +import meerkat.comm.MessageInputStream.MessageInputStreamFactory; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Crypto; +import meerkat.util.BulletinBoardMessageComparator; +import org.junit.Test; + +import java.io.*; +import java.lang.reflect.InvocationTargetException; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + * Tests for MessageInputStream and MessageOutputStream classes + */ +public class MessageStreamTest { + + @Test + public void testWithBulletinBoardMessages() { + + MessageOutputStream out; + MessageInputStream in; + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + BulletinBoardMessageComparator comparator = new BulletinBoardMessageComparator(); + + byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4}; + byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14}; + byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24}; + + try { + + out = new MessageOutputStream<>(stream); + + } catch (IOException e) { + + System.err.println(e.getMessage()); + assertThat("Error creating streams: " + e.getMessage(), false); + return; + + } + + + + BulletinBoardMessage message = BulletinBoardMessage.newBuilder() + .setEntryNum(1) + .setMsg(UnsignedBulletinBoardMessage.newBuilder() + .setData(ByteString.copyFrom(b1)) + .addTag("Test") + .addTag("1234") + .setTimestamp(com.google.protobuf.Timestamp.newBuilder() + .setSeconds(19823451) + .setNanos(2134) + .build()) + .build()) + .addSig(Crypto.Signature.newBuilder() + .setSignerId(ByteString.copyFrom(b2)) + .setData(ByteString.copyFrom(b3)) + .build()) + .build(); + + try { + + out.writeMessage(message); + + } catch (IOException e) { + + System.err.println(e.getMessage()); + assertThat("Error writing message: " + e.getMessage(), false); + + } + + try { + + in = MessageInputStreamFactory.createMessageInputStream( + new ByteArrayInputStream(stream.toByteArray()), + BulletinBoardMessage.class); + + assertThat("Retrieved message was not identical to send message", comparator.compare(message, in.readMessage()), is(equalTo(0))); + + } catch (IOException e) { + + System.err.println(e.getMessage()); + assertThat("Error reading message: " + e.getMessage(), false); + + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + System.err.println(e.getMessage()); + assertThat("Error creating input stream " + e.getMessage(), false); + } + + } + +} From 71191e05b9ab8c807d6fcad8487757f5fbd43aca Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Mon, 29 Feb 2016 08:36:35 +0200 Subject: [PATCH 11/12] Added Sync Query tests on Bulletin Board Server --- .../sqlserver/SQLiteQueryProvider.java | 92 +++++++++++++++++++ .../GenericBulletinBoardServerTest.java | 88 +++++++++++++++++- .../MySQLBulletinBoardServerTest.java | 13 +++ .../java/meerkat/comm/MessageInputStream.java | 30 +++++- .../java/meerkat/util/BulletinBoardUtils.java | 2 +- 5 files changed, 219 insertions(+), 6 deletions(-) diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java index b581b87..b64412d 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java @@ -4,6 +4,7 @@ import meerkat.protobuf.BulletinBoardAPI.*; import org.sqlite.SQLiteDataSource; import javax.sql.DataSource; +import java.text.MessageFormat; import java.util.LinkedList; import java.util.List; @@ -25,19 +26,97 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi switch(queryType) { case ADD_SIGNATURE: return "INSERT OR IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:EntryNum,:SignerId,:Signature)"; + case CONNECT_TAG: return "INSERT OR IGNORE INTO MsgTagTable (TagId, EntryNum)" + " SELECT TagTable.TagId, :EntryNum AS EntryNum FROM TagTable WHERE Tag = :Tag"; + case FIND_MSG_ID: return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId"; + + case FIND_TAG_ID: + return MessageFormat.format( + "SELECT TagId FROM TagTable WHERE Tag = :{0}", + QueryType.FIND_TAG_ID.getParamName(0)); + case GET_MESSAGES: return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"; + + case COUNT_MESSAGES: + return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable"; + + case GET_MESSAGE_STUBS: + return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable"; + case GET_SIGNATURES: return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum"; + case INSERT_MSG: return "INSERT INTO MsgTable (MsgId, Msg) VALUES(:MsgId,:Msg)"; + case INSERT_NEW_TAG: return "INSERT OR IGNORE INTO TagTable(Tag) VALUES (:Tag)"; + + case GET_LAST_MESSAGE_ENTRY: + return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable"; + + case GET_BATCH_MESSAGE_ENTRY: + return MessageFormat.format( + "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable" + + " INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum" + + " INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum" + + " INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId" + + " WHERE SignatureTable.SignerId = :{0}" + + " AND TagTable.Tag = :{1}", + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0), + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1)); + + case GET_BATCH_MESSAGE_DATA: + return MessageFormat.format( + "SELECT Data FROM BatchTable" + + " WHERE SignerId = :{0} AND BatchId = :{1} AND SerialNum >= :{2}" + + " ORDER BY SerialNum ASC", + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2)); + + case INSERT_BATCH_DATA: + return MessageFormat.format( + "INSERT INTO BatchTable (SignerId, BatchId, SerialNum, Data)" + + " VALUES (:{0}, :{1}, :{2}, :{3})", + QueryType.INSERT_BATCH_DATA.getParamName(0), + QueryType.INSERT_BATCH_DATA.getParamName(1), + QueryType.INSERT_BATCH_DATA.getParamName(2), + QueryType.INSERT_BATCH_DATA.getParamName(3)); + + case CHECK_BATCH_LENGTH: + return MessageFormat.format( + "SELECT COUNT(Data) AS BatchLength FROM BatchTable" + + " WHERE SignerId = :{0} AND BatchId = :{1}", + QueryType.CHECK_BATCH_LENGTH.getParamName(0), + QueryType.CHECK_BATCH_LENGTH.getParamName(1)); + + case CONNECT_BATCH_TAG: + return MessageFormat.format( + "INSERT INTO BatchTagTable (SignerId, BatchId, TagId) SELECT :{0}, :{1}, TagId FROM TagTable" + + " WHERE Tag = :{2}", + QueryType.CONNECT_BATCH_TAG.getParamName(0), + QueryType.CONNECT_BATCH_TAG.getParamName(1), + QueryType.CONNECT_BATCH_TAG.getParamName(2)); + + case GET_BATCH_TAGS: + return MessageFormat.format( + "SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.TagId" + + " WHERE SignerId = :{0} AND BatchId = :{1} ORDER BY Tag ASC", + QueryType.GET_BATCH_TAGS.getParamName(0), + QueryType.GET_BATCH_TAGS.getParamName(1)); + + case REMOVE_BATCH_TAGS: + return MessageFormat.format( + "DELETE FROM BatchTagTable WHERE SignerId = :{0} AND BatchId = :{1}", + QueryType.REMOVE_BATCH_TAGS.getParamName(0), + QueryType.REMOVE_BATCH_TAGS.getParamName(1)); + default: throw new IllegalArgumentException("Cannot serve a query of type " + queryType); } @@ -52,21 +131,34 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi switch(filterType) { case EXACT_ENTRY: return "MsgTable.EntryNum = :EntryNum" + serialString; + case MAX_ENTRY: return "MsgTable.EntryNum <= :EntryNum" + serialString; + case MIN_ENTRY: return "MsgTable.EntryNum <= :EntryNum" + serialString; + case MAX_MESSAGES: return "LIMIT = :Limit" + serialString; + case MSG_ID: return "MsgTable.MsgId = :MsgId" + serialString; + case SIGNER_ID: return "EXISTS (SELECT 1 FROM SignatureTable" + " WHERE SignatureTable.SignerId = :SignerId" + serialString + " AND SignatureTable.EntryNum = MsgTable.EntryNum)"; + case TAG: return "EXISTS (SELECT 1 FROM TagTable" + " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId" + " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)"; + + case BEFORE_TIME: + return "MsgTable.ExactTime <= :TimeStamp"; + + case AFTER_TIME: + return "MsgTable.ExactTime >= :TimeStamp"; + default: throw new IllegalArgumentException("Cannot serve a filter of type " + filterType); } diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java index e33c739..1e7921d 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java @@ -24,9 +24,12 @@ import meerkat.comm.CommunicationException; import meerkat.comm.MessageInputStream; import meerkat.comm.MessageOutputStream; import meerkat.comm.MessageInputStream.MessageInputStreamFactory; +import meerkat.crypto.Digest; import meerkat.crypto.concrete.ECDSASignature; +import meerkat.crypto.concrete.SHA256Digest; import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.util.BulletinBoardUtils; +import meerkat.util.BulletinBoardMessageGenerator; +import org.h2.util.DateTimeUtils; import static org.junit.Assert.*; import static org.hamcrest.CoreMatchers.*; @@ -59,6 +62,10 @@ public class GenericBulletinBoardServerTest { private final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); // Used to time the tests + private BulletinBoardMessageGenerator bulletinBoardMessageGenerator; + + private Digest digest; + /** * @param bulletinBoardServer is an initialized server. * @throws InstantiationException @@ -121,8 +128,12 @@ public class GenericBulletinBoardServerTest { System.err.println("Couldn't find signing key " + e.getMessage()); fail("Couldn't find signing key " + e.getMessage()); } - - random = new Random(0); // We use insecure randomness in tests for repeatability + + // We use insecure randomness in tests for repeatability + random = new Random(0); + bulletinBoardMessageGenerator = new BulletinBoardMessageGenerator(random); + + digest = new SHA256Digest(); long end = threadBean.getCurrentThreadCpuTime(); System.err.println("Finished initializing GenericBulletinBoardServerTest"); @@ -182,7 +193,10 @@ public class GenericBulletinBoardServerTest { for (i = 1; i <= MESSAGE_NUM; i++) { unsignedMsgBuilder = UnsignedBulletinBoardMessage.newBuilder() .setData(ByteString.copyFrom(data[i - 1])) - .setTimestamp(BulletinBoardUtils.toTimestampProto()); + .setTimestamp(Timestamp.newBuilder() + .setSeconds(i) + .setNanos(i) + .build()); // Add tags based on bit-representation of message number. @@ -619,6 +633,72 @@ public class GenericBulletinBoardServerTest { } } + + public void testSyncQuery() + throws SignatureException, CommunicationException, IOException,NoSuchMethodException, IllegalAccessException, InvocationTargetException { + + Timestamp timestamp = Timestamp.newBuilder() + .setSeconds(1) + .setNanos(0) + .build(); + + BulletinBoardMessage newMessage = bulletinBoardMessageGenerator.generateRandomMessage(signers, timestamp, 10, 10); + + BoolMsg result = bulletinBoardServer.postMessage(newMessage); + assertThat("Failed to post message to BB Server", result.getValue(), is(true)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + digest.update(newMessage.getMsg()); + + ByteString messageID = ByteString.copyFrom(digest.digest()); + + MessageFilterList filterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.MSG_ID) + .setId(messageID) + .build()) + .build(); + + bulletinBoardServer.readMessages(filterList, new MessageOutputStream(outputStream)); + + MessageInputStream inputStream = + MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream( + outputStream.toByteArray()), + BulletinBoardMessage.class); + + long lastEntry = inputStream.asList().get(0).getEntryNum(); + + SyncQuery syncQuery = SyncQuery.newBuilder() + .setFilterList(MessageFilterList.getDefaultInstance()) + .addQuery(SingleSyncQuery.newBuilder() + .setChecksum(2) + .setTimeOfSync(Timestamp.newBuilder() + .setSeconds(2) + .setNanos(0) + .build()) + .build()) + .build(); + + SyncQueryResponse queryResponse = bulletinBoardServer.querySync(syncQuery); + + assertThat("Sync query replies with positive sync when no sync was expected", queryResponse.getLastEntryNum(), is(equalTo(-1l))); + + syncQuery = SyncQuery.newBuilder() + .setFilterList(MessageFilterList.getDefaultInstance()) + .addQuery(SingleSyncQuery.newBuilder() + .setChecksum(messageID.byteAt(0) & messageID.byteAt(1) & messageID.byteAt(2) & messageID.byteAt(3)) + .setTimeOfSync(timestamp) + .build()) + .build(); + + queryResponse = bulletinBoardServer.querySync(syncQuery); + + assertThat("Sync query reply contained wrong last entry number", lastEntry, is(equalTo(queryResponse.getLastEntryNum()))); + + assertThat("Sync query reply contained wrong timestamp", timestamp, is(equalTo(queryResponse.getLastTimeOfSync()))); + + } public void close(){ signers[0].clearSigningKey(); diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java index 42c11f7..273a53f 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java @@ -8,8 +8,11 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; +import java.lang.reflect.InvocationTargetException; +import java.security.SignatureException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -144,6 +147,16 @@ public class MySQLBulletinBoardServerTest { } + @Test + public void testSyncQuery() { + try { + serverTest.testSyncQuery(); + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + } + @After public void close() { System.err.println("Starting to close MySQLBulletinBoardServerTest"); diff --git a/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java b/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java index 33deb3f..dc637d9 100644 --- a/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java +++ b/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java @@ -5,6 +5,7 @@ import com.google.protobuf.Message; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -12,7 +13,7 @@ import java.util.List; * Created by Arbel Deutsch Peled on 21-Feb-16. * A input stream of Protobuf messages */ -public class MessageInputStream{ +public class MessageInputStream implements Iterable{ private T.Builder builder; @@ -23,6 +24,33 @@ public class MessageInputStream{ this.builder = (T.Builder) type.getMethod("newBuilder").invoke(type); } + @Override + public Iterator iterator() { + + return new Iterator() { + + @Override + public boolean hasNext() { + try { + return isAvailable(); + } catch (IOException e) { + return false; + } + } + + @Override + public T next() { + try { + return readMessage(); + } catch (IOException e) { + return null; + } + } + + }; + + } + /** * Factory class for actually creating a MessageInputStream */ diff --git a/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java b/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java index d8b362c..8793b2d 100644 --- a/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java +++ b/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java @@ -80,7 +80,7 @@ public class BulletinBoardUtils { * This method creates a Timestamp Protobuf from the current system time * @return a Timestamp Protobuf encoding of the current system time */ - public static com.google.protobuf.Timestamp toTimestampProto() { + public static com.google.protobuf.Timestamp getCurrentTimestampProto() { return toTimestampProto(System.currentTimeMillis()); From 1cf14a60a805a17ebcc795d64a990317c4b414a3 Mon Sep 17 00:00:00 2001 From: Arbel Deutsch Peled Date: Tue, 1 Mar 2016 13:56:18 +0200 Subject: [PATCH 12/12] Bulletin Board Client support for streaming and Timestamps Created standard Checksum interface and implementation for Sync Query mechanism Added the Timestamp into the Batch Digest and Signature logic --- .../SingleServerBulletinBoardClient.java | 6 +- .../SingleServerReadBatchWorker.java | 30 ++-- .../SingleServerReadMessagesWorker.java | 32 +++-- ...dedBulletinBoardClientIntegrationTest.java | 20 ++- .../sqlserver/BulletinBoardSQLServer.java | 20 +-- .../GenericBulletinBoardServerTest.java | 4 +- .../java/meerkat/bulletinboard/Checksum.java | 122 ++++++++++++++++ .../bulletinboard/GenericBatchDigest.java | 2 + .../GenericBatchDigitalSignature.java | 3 + .../meerkat/bulletinboard/SimpleChecksum.java | 131 ++++++++++++++++++ .../java/meerkat/comm/MessageInputStream.java | 6 + .../meerkat/comm/MessageOutputStream.java | 4 + 12 files changed, 342 insertions(+), 38 deletions(-) create mode 100644 meerkat-common/src/main/java/meerkat/bulletinboard/Checksum.java create mode 100644 meerkat-common/src/main/java/meerkat/bulletinboard/SimpleChecksum.java diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java index ea0e93d..24cd6bf 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java @@ -381,11 +381,7 @@ public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient i @Override public void onSuccess(Boolean msg) { closeBatch( - CloseBatchMessage.newBuilder() - .setBatchId(completeBatch.getBeginBatchMessage().getBatchId()) - .setSig(completeBatch.getSignature()) - .setBatchLength(completeBatch.getBatchDataList().size()) - .build(), + completeBatch.getCloseBatchMessage(), callback ); } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java index 11fc777..57b336e 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java @@ -3,6 +3,7 @@ package meerkat.bulletinboard.workers.singleserver; import meerkat.bulletinboard.CompleteBatch; import meerkat.bulletinboard.SingleServerWorker; import meerkat.comm.CommunicationException; +import meerkat.comm.MessageInputStream; import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.rest.Constants; @@ -12,6 +13,9 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.GenericType; import javax.ws.rs.core.Response; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; import java.util.List; import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; @@ -40,29 +44,33 @@ public class SingleServerReadBatchWorker extends SingleServerWorker inputStream = null; try { - // If a BatchDataList is returned: the read was successful - return response.readEntity(BatchDataList.class).getDataList(); + inputStream = MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BatchData.class); - } catch (ProcessingException | IllegalStateException e) { + return inputStream.asList(); + + } catch (IOException | InvocationTargetException e) { // Read failed - throw new CommunicationException("Could not contact the server"); + throw new CommunicationException("Could not contact the server or server returned illegal result"); - } - finally { - response.close(); + } catch (NoSuchMethodException | IllegalAccessException e) { + + throw new CommunicationException("MessageInputStream error"); + + } finally { + try { + inputStream.close(); + } catch (IOException ignored) {} } } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java index 6c09bcc..d8525ab 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java @@ -2,6 +2,8 @@ package meerkat.bulletinboard.workers.singleserver; import meerkat.bulletinboard.SingleServerWorker; import meerkat.comm.CommunicationException; +import meerkat.comm.MessageInputStream; +import meerkat.protobuf.BulletinBoardAPI; import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessageList; import meerkat.protobuf.BulletinBoardAPI.MessageFilterList; import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; @@ -13,6 +15,9 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; import java.util.List; import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; @@ -38,30 +43,33 @@ public class SingleServerReadMessagesWorker extends SingleServerWorker inputStream = null; try { - // If a BulletinBoardMessageList is returned: the read was successful - return response.readEntity(BulletinBoardMessageList.class).getMessageList(); + inputStream = MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BulletinBoardMessage.class); - } catch (ProcessingException | IllegalStateException e) { + return inputStream.asList(); + + } catch (IOException | InvocationTargetException e) { // Read failed - throw new CommunicationException("Could not contact the server"); + throw new CommunicationException("Could not contact the server or server returned illegal result"); - } - finally { - response.close(); - } + } catch (NoSuchMethodException | IllegalAccessException e) { + throw new CommunicationException("MessageInputStream error"); + + } finally { + try { + inputStream.close(); + } catch (IOException ignored) {} + } } diff --git a/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java b/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java index dfccebc..59aa615 100644 --- a/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java +++ b/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java @@ -1,5 +1,6 @@ import com.google.common.util.concurrent.FutureCallback; -import com.google.protobuf.ByteString; +import com.google.protobuf.*; +import com.google.protobuf.Timestamp; import meerkat.bulletinboard.AsyncBulletinBoardClient; import meerkat.bulletinboard.CompleteBatch; import meerkat.bulletinboard.GenericBatchDigitalSignature; @@ -284,6 +285,11 @@ public class ThreadedBulletinBoardClientIntegrationTest { } + completeBatch.setTimestamp(Timestamp.newBuilder() + .setSeconds(Math.abs(90)) + .setNanos(50) + .build()); + signers[signer].updateContent(completeBatch); completeBatch.setSignature(signers[signer].sign()); @@ -357,6 +363,10 @@ public class ThreadedBulletinBoardClientIntegrationTest { .addTag("Signature") .addTag("Trustee") .setData(ByteString.copyFrom(b1)) + .setTimestamp(Timestamp.newBuilder() + .setSeconds(20) + .setNanos(30) + .build()) .build()) .addSig(Crypto.Signature.newBuilder() .setType(Crypto.SignatureType.DSA) @@ -440,6 +450,10 @@ public class ThreadedBulletinBoardClientIntegrationTest { CloseBatchMessage closeBatchMessage = CloseBatchMessage.newBuilder() .setBatchId(BATCH_ID) .setBatchLength(BATCH_LENGTH) + .setTimestamp(Timestamp.newBuilder() + .setSeconds(50) + .setNanos(80) + .build()) .setSig(completeBatch.getSignature()) .build(); @@ -525,6 +539,10 @@ public class ThreadedBulletinBoardClientIntegrationTest { .setBatchId(NON_EXISTENT_BATCH_ID) .setBatchLength(1) .setSig(Crypto.Signature.getDefaultInstance()) + .setTimestamp(Timestamp.newBuilder() + .setSeconds(9) + .setNanos(12) + .build()) .build(); // Try to close the (unopened) batch; diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java index b4e2949..c7bcf57 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java @@ -446,9 +446,10 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ sql = sqlQueryProvider.getSQLString(QueryType.FIND_MSG_ID); Map namedParameters = new HashMap(); + namedParameters.put(QueryType.FIND_MSG_ID.getParamName(0),msgID); - List entryNums = jdbcTemplate.query(sql, new MapSqlParameterSource(namedParameters), new LongMapper()); + List entryNums = jdbcTemplate.query(sql, namedParameters, new LongMapper()); if (entryNums.size() > 0){ @@ -462,7 +463,7 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ namedParameters.put(QueryType.INSERT_MSG.getParamName(2), msg.getMsg().toByteArray()); KeyHolder keyHolder = new GeneratedKeyHolder(); - jdbcTemplate.update(sql,new MapSqlParameterSource(namedParameters),keyHolder); + jdbcTemplate.update(sql, new MapSqlParameterSource(namedParameters), keyHolder); entryNum = keyHolder.getKey().longValue(); @@ -785,6 +786,9 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ .build() ); + // Add timestamp to CompleteBatch + completeBatch.setTimestamp(message.getTimestamp()); + // Add actual batch data to CompleteBatch sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA); @@ -903,20 +907,20 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ long lastEntryNum = getLastMessageEntry(); - long checksum = 0; - Iterator queryIterator = syncQuery.getQueryList().iterator(); SingleSyncQuery currentQuery = queryIterator.next(); List messageStubs = readMessageStubs(syncQuery.getFilterList()); + Checksum checksum = new SimpleChecksum(); + for (BulletinBoardMessage message : messageStubs){ // Check for end of current query if (timestampComparator.compare(message.getMsg().getTimestamp(), currentQuery.getTimeOfSync()) > 0){ - if (checksum == currentQuery.getChecksum()){ + if (checksum.getChecksum() == currentQuery.getChecksum()){ lastTimeOfSync = currentQuery.getTimeOfSync(); } else { break; @@ -932,13 +936,13 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ // Advance checksum - ByteString messageID = message.getMsg().getData(); + ByteString messageID = message.getMsg().getData(); // The data field contains the message ID - checksum &= messageID.byteAt(0) & messageID.byteAt(1) & messageID.byteAt(2) & messageID.byteAt(3); + checksum.update(messageID); } - if (checksum == currentQuery.getChecksum()){ + if (checksum.getChecksum() == currentQuery.getChecksum()){ lastTimeOfSync = currentQuery.getTimeOfSync(); } diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java index 1e7921d..affb1a3 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java @@ -637,6 +637,8 @@ public class GenericBulletinBoardServerTest { public void testSyncQuery() throws SignatureException, CommunicationException, IOException,NoSuchMethodException, IllegalAccessException, InvocationTargetException { + Checksum checksum = new SimpleChecksum(); + Timestamp timestamp = Timestamp.newBuilder() .setSeconds(1) .setNanos(0) @@ -687,7 +689,7 @@ public class GenericBulletinBoardServerTest { syncQuery = SyncQuery.newBuilder() .setFilterList(MessageFilterList.getDefaultInstance()) .addQuery(SingleSyncQuery.newBuilder() - .setChecksum(messageID.byteAt(0) & messageID.byteAt(1) & messageID.byteAt(2) & messageID.byteAt(3)) + .setChecksum(checksum.getChecksum(messageID)) .setTimeOfSync(timestamp) .build()) .build(); diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/Checksum.java b/meerkat-common/src/main/java/meerkat/bulletinboard/Checksum.java new file mode 100644 index 0000000..b8ddb0b --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/Checksum.java @@ -0,0 +1,122 @@ +package meerkat.bulletinboard; + +import com.google.protobuf.ByteString; +import meerkat.crypto.Digest; +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; +import meerkat.protobuf.BulletinBoardAPI.MessageID; + +import java.util.Collection; + +/** + * Created by Arbel Deutsch Peled on 01-Mar-16. + * This interface is used to create checksums of Bulletin Board messages IDs + * This is useful in comparing database states + */ +public interface Checksum { + + /** + * Sets the Digest method which is used in creating message IDs from the messages themselves + * This method must be called with an initialized Digest before calling any methods that receive a parameter of type BulletinBoardMessage + * @param digest is the Digest that will be used to create message IDs from Bulletin Board Messages + */ + public void setDigest(Digest digest); + + /** + * Used to reset the current checksum state + */ + public void reset(); + + /** + * Update the current checksum with the given ID + * @param messageID is the message ID to be added + */ + public void update(MessageID messageID); + + /** + * Update the current checksum with the given collection of IDs + * @param messageIDs contains the message IDs + */ + public void update(Collection messageIDs); + + /** + * Update the current checksum with the given ID + * @param messageID is the message ID to be added + */ + public void update(ByteString messageID); + + /** + * Update the current checksum with the given ID + * @param messageID is the message ID to be added + */ + public void update(byte[] messageID); + + /** + * Update the current checksum with the message ID of the given message + * @param bulletinBoardMessage is the message whose ID should be added to the checksum + * @throws IllegalStateException if a Digest has not been set before calling this method + */ + public void digestAndUpdate(BulletinBoardMessage bulletinBoardMessage) throws IllegalStateException; + + + /** + * Update the current checksum with the message IDs of the given messages + * @param bulletinBoardMessages contains the messages whose IDs should be added to the checksum + * @throws IllegalStateException if a Digest has not been set before calling this method + */ + public void digestAndUpdate(Collection bulletinBoardMessages) throws IllegalStateException; + + /** + * Returns the current checksum without changing the checksum state + * @return the current checksum + */ + public long getChecksum(); + + /** + * Updates the current checksum with the given ID and returns the resulting checksum + * The checksum is not reset afterwards + * @param messageID is the message ID to be added + * @return the updated checksum + */ + public long getChecksum(MessageID messageID); + + /** + * Updates the current checksum with the given ID and returns the resulting checksum + * The checksum is not reset afterwards + * @param messageID is the message ID to be added + * @return the updated checksum + */ + public long getChecksum(ByteString messageID); + + /** + * Updates the current checksum with the given ID and returns the resulting checksum + * The checksum is not reset afterwards + * @param messageID is the message ID to be added + * @return the updated checksum + */ + public long getChecksum(byte[] messageID); + + /** + * Updates the current checksum with the given IDs and returns the resulting checksum + * The checksum is not reset afterwards + * @param messageIDs contains the message IDs to be added + * @return the updated checksum + */ + public long getChecksum(Collection messageIDs); + + /** + * Updates the current checksum with the message ID of the given message + * The checksum is not reset afterwards + * @param bulletinBoardMessage is the message whose ID should be added to the checksum + * @return the updated checksum + */ + public long digestAndGetChecksum(BulletinBoardMessage bulletinBoardMessage) throws IllegalStateException; + + /** + * Updates the current checksum with the message IDs of the given messages + * The checksum is not reset afterwards + * @param bulletinBoardMessages contains the messages whose IDs should be added to the checksum + * @return the updated checksum + */ + public long digestAndGetChecksum(Collection bulletinBoardMessages) throws IllegalStateException; + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java index 4f25f59..852bb24 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java @@ -29,6 +29,8 @@ public class GenericBatchDigest implements BatchDigest{ update(batchData); } + update(completeBatch.getTimestamp()); + } @Override diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java index 7174b42..7327b8e 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java @@ -32,10 +32,13 @@ public class GenericBatchDigitalSignature implements BatchDigitalSignature{ public void updateContent(CompleteBatch completeBatch) throws SignatureException { digitalSignature.updateContent(completeBatch.getBeginBatchMessage()); + for (BatchData batchData : completeBatch.getBatchDataList()) { digitalSignature.updateContent(batchData); } + digitalSignature.updateContent(completeBatch.getTimestamp()); + } @Override diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/SimpleChecksum.java b/meerkat-common/src/main/java/meerkat/bulletinboard/SimpleChecksum.java new file mode 100644 index 0000000..90d7ae5 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/SimpleChecksum.java @@ -0,0 +1,131 @@ +package meerkat.bulletinboard; + +import com.google.protobuf.ByteString; +import meerkat.crypto.Digest; +import meerkat.protobuf.BulletinBoardAPI.MessageID; +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; + +import java.util.Collection; + +/** + * Created by Arbel Deutsch Peled on 01-Mar-16. + * Implementation of Checksum via bitwise XOR of the bytes of message IDs + */ +public class SimpleChecksum implements Checksum{ + + private Digest digest; + private long checksum; + + public SimpleChecksum() { + digest = null; + reset(); + } + + @Override + public void setDigest(Digest digest) { + this.digest = digest; + } + + @Override + public void reset() { + checksum = 0; + } + + @Override + public void update(MessageID messageID) { + ByteString messageIDByteString = messageID.getID(); + update(messageIDByteString); + } + + @Override + public void update(Collection messageIDs) { + + for (MessageID messageID : messageIDs){ + update(messageID); + } + + } + + @Override + public void update(ByteString messageID) { + for (int i = 0 ; i < messageID.size() ; i++){ + checksum &= messageID.byteAt(i); + } + } + + @Override + public void update(byte[] messageID) { + for (int i = 0 ; i < messageID.length ; i++){ + checksum &= messageID[i]; + } + } + + private void checkDigest() throws IllegalStateException { + + if (digest == null){ + throw new IllegalStateException("Digest method not set. Use setDigest method before calling digestAndUpdate."); + } + + } + + @Override + public void digestAndUpdate(BulletinBoardMessage bulletinBoardMessage) throws IllegalStateException { + + checkDigest(); + + digest.reset(); + digest.update(bulletinBoardMessage); + update(digest.digest()); + + } + + @Override + public void digestAndUpdate(Collection bulletinBoardMessages) throws IllegalStateException { + + for (BulletinBoardMessage bulletinBoardMessage : bulletinBoardMessages){ + digestAndUpdate(bulletinBoardMessage); + } + + } + + @Override + public long getChecksum() { + return checksum; + } + + @Override + public long getChecksum(MessageID messageID) { + update(messageID); + return getChecksum(); + } + + @Override + public long getChecksum(ByteString messageID) { + update(messageID); + return getChecksum(); + } + + @Override + public long getChecksum(byte[] messageID) { + update(messageID); + return getChecksum(); + } + + @Override + public long getChecksum(Collection messageIDs) { + update(messageIDs); + return getChecksum(); + } + + @Override + public long digestAndGetChecksum(BulletinBoardMessage bulletinBoardMessage) throws IllegalStateException { + digestAndUpdate(bulletinBoardMessage); + return getChecksum(); + } + + @Override + public long digestAndGetChecksum(Collection bulletinBoardMessages) throws IllegalStateException { + digestAndUpdate(bulletinBoardMessages); + return getChecksum(); + } +} diff --git a/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java b/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java index dc637d9..b1e5255 100644 --- a/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java +++ b/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java @@ -89,4 +89,10 @@ public class MessageInputStream implements Iterable{ } + public void close() throws IOException { + + in.close(); + + } + } diff --git a/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java b/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java index b55bc8e..f72c04c 100644 --- a/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java +++ b/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java @@ -21,4 +21,8 @@ public class MessageOutputStream { message.writeDelimitedTo(out); } + public void close() throws IOException { + out.close(); + } + }