diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BatchDataContainer.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BatchDataContainer.java new file mode 100644 index 0000000..53f4870 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BatchDataContainer.java @@ -0,0 +1,25 @@ +package meerkat.bulletinboard; + +import com.google.protobuf.ByteString; +import meerkat.protobuf.BulletinBoardAPI.BatchData; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 17-Jan-16. + * Used to store the complete data required for sending a batch data list inside a single object + */ +public class BatchDataContainer { + + public final byte[] signerId; + public final int batchId; + public final List batchDataList; + public final int startPosition; + + public BatchDataContainer(byte[] signerId, int batchId, List batchDataList, int startPosition) { + this.signerId = signerId; + this.batchId = batchId; + this.batchDataList = batchDataList; + this.startPosition = startPosition; + } +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJob.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJob.java deleted file mode 100644 index aca98d4..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJob.java +++ /dev/null @@ -1,82 +0,0 @@ -package meerkat.bulletinboard; - -import com.google.protobuf.Message; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -/** - * Created by Arbel Deutsch Peled on 09-Dec-15. - * - * This class specifies the job that is required of a Bulletin Board Client Worker - */ -public class BulletinClientJob { - - public static enum JobType{ - POST_MESSAGE, // Post a message to servers - READ_MESSAGES, // Read messages according to some given filter (any server will do) - GET_REDUNDANCY // Check the redundancy of a specific message in the databases - } - - private List serverAddresses; - - private int minServers; // The minimal number of servers the job must be successful on for the job to be completed - - private final JobType jobType; - - private final Message payload; // The information associated with the job type - - private int maxRetry; // Number of retries for this job; set to -1 for infinite retries - - public BulletinClientJob(List serverAddresses, int minServers, JobType jobType, Message payload, int maxRetry) { - this.serverAddresses = serverAddresses; - this.minServers = minServers; - this.jobType = jobType; - this.payload = payload; - this.maxRetry = maxRetry; - } - - public void updateServerAddresses(List newServerAdresses) { - this.serverAddresses = newServerAdresses; - } - - public List getServerAddresses() { - return serverAddresses; - } - - public int getMinServers() { - return minServers; - } - - public JobType getJobType() { - return jobType; - } - - public Message getPayload() { - return payload; - } - - public int getMaxRetry() { - return maxRetry; - } - - public void shuffleAddresses() { - Collections.shuffle(serverAddresses); - } - - public void decMinServers(){ - minServers--; - } - - public void decMaxRetry(){ - if (maxRetry > 0) { - maxRetry--; - } - } - - public boolean isRetry(){ - return (maxRetry != 0); - } - -} \ No newline at end of file diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJobResult.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJobResult.java deleted file mode 100644 index be0501b..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJobResult.java +++ /dev/null @@ -1,29 +0,0 @@ -package meerkat.bulletinboard; - -import com.google.protobuf.Message; - -/** - * Created by Arbel Deutsch Peled on 09-Dec-15. - * - * This class contains the end status and result of a Bulletin Board Client Job. - */ -public final class BulletinClientJobResult { - - private final BulletinClientJob job; // Stores the job the result refers to - - private final Message result; // The result of the job; valid only if success==true - - public BulletinClientJobResult(BulletinClientJob job, Message result) { - this.job = job; - this.result = result; - } - - public BulletinClientJob getJob() { - return job; - } - - public Message getResult() { - return result; - } - -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java index 51599d5..1a4b62f 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java @@ -1,217 +1,38 @@ package meerkat.bulletinboard; -import com.google.protobuf.Message; -import meerkat.comm.CommunicationException; -import meerkat.crypto.Digest; -import meerkat.crypto.concrete.SHA256Digest; -import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.rest.Constants; -import meerkat.rest.ProtobufMessageBodyReader; -import meerkat.rest.ProtobufMessageBodyWriter; - -import javax.ws.rs.ProcessingException; -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.Response; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.Callable; - /** * Created by Arbel Deutsch Peled on 09-Dec-15. * - * This class implements the actual communication with the Bulletin Board Servers. + * This class handles bulletin client work. * It is meant to be used in a multi-threaded environment. */ -//TODO: Maybe make this abstract and inherit from it. -public class BulletinClientWorker implements Callable { +public abstract class BulletinClientWorker { - private final BulletinClientJob job; // The requested job to be handled + protected final IN payload; // Payload of the job - public BulletinClientWorker(BulletinClientJob job){ - this.job = job; + private int maxRetry; // Number of retries for this job; set to -1 for infinite retries + + public BulletinClientWorker(IN payload, int maxRetry) { + this.payload = payload; + this.maxRetry = maxRetry; } - // This resource enabled creation of a single Client per thread. - private static final ThreadLocal clientLocal = - new ThreadLocal () { - @Override protected Client initialValue() { - Client client; - client = ClientBuilder.newClient(); - client.register(ProtobufMessageBodyReader.class); - client.register(ProtobufMessageBodyWriter.class); + public IN getPayload() { + return payload; + } - return client; - } - }; - - // This resource enables creation of a single Digest per thread. - private static final ThreadLocal digestLocal = - new ThreadLocal () { - @Override protected Digest initialValue() { - Digest digest; - digest = new SHA256Digest(); //TODO: Make this generic. - - return digest; - } - }; - - /** - * This method carries out the actual communication with the servers via HTTP Post - * It accesses the servers according to the job it received and updates said job as it goes - * The method will only iterate once through the server list, removing servers from the list when they are no longer required - * In a POST_MESSAGE job: successful post to a server results in removing the server from the list - * In a GET_REDUNDANCY job: no server is removed from the list and the (absolute) number of servers in which the message was found is returned - * In a READ_MESSAGES job: successful retrieval from any server terminates the method and returns the received values; The list is not changed - * @return The original job, modified to fit the current state and the required output (if any) of the operation - * @throws IllegalArgumentException - * @throws CommunicationException - */ - public BulletinClientJobResult call() throws IllegalArgumentException, CommunicationException{ - - Client client = clientLocal.get(); - Digest digest = digestLocal.get(); - - WebTarget webTarget; - Response response; - - String requestPath; - Message msg; - - List serverAddresses = new LinkedList(job.getServerAddresses()); - - Message payload = job.getPayload(); - - BulletinBoardMessageList msgList; - - int count = 0; // Used to count number of servers which contain the required message in a GET_REDUNDANCY request. - - job.shuffleAddresses(); // This is done to randomize the order of access to servers primarily for READ operations - - // Prepare the request. - switch(job.getJobType()) { - - case POST_MESSAGE: - // Make sure the payload is a BulletinBoardMessage - if (!(payload instanceof BulletinBoardMessage)) { - throw new IllegalArgumentException("Cannot post an object that is not an instance of BulletinBoardMessage"); - } - - msg = payload; - requestPath = Constants.POST_MESSAGE_PATH; - break; - - case READ_MESSAGES: - // Make sure the payload is a MessageFilterList - if (!(payload instanceof MessageFilterList)) { - throw new IllegalArgumentException("Read failed: an instance of MessageFilterList is required as payload for a READ_MESSAGES operation"); - } - - msg = payload; - requestPath = Constants.READ_MESSAGES_PATH; - break; - - case GET_REDUNDANCY: - // Make sure the payload is a MessageId - if (!(payload instanceof MessageID)) { - throw new IllegalArgumentException("Cannot search for an object that is not an instance of MessageID"); - } - - requestPath = Constants.READ_MESSAGES_PATH; - - msg = MessageFilterList.newBuilder() - .addFilter(MessageFilter.newBuilder() - .setType(FilterType.MSG_ID) - .setId(((MessageID) payload).getID()) - .build() - ).build(); - - break; - - default: - throw new IllegalArgumentException("Unsupported job type"); - - } - - // Iterate through servers - - Iterator addressIterator = serverAddresses.iterator(); - - while (addressIterator.hasNext()) { - - // Send request to Server - String address = addressIterator.next(); - webTarget = client.target(address).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(requestPath); - response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF)); - - // Retrieve answer - switch(job.getJobType()) { - - case POST_MESSAGE: - try { - - response.readEntity(BoolMsg.class); // If a BoolMsg entity is returned: the post was successful - addressIterator.remove(); // Post to this server succeeded: remove server from list - job.decMinServers(); - - } catch (ProcessingException | IllegalStateException e) {} // Post to this server failed: retry next time - finally { - response.close(); - } - break; - - case GET_REDUNDANCY: - try { - msgList = response.readEntity(BulletinBoardMessageList.class); // If a BulletinBoardMessageList is returned: the read was successful - - if (msgList.getMessageList().size() > 0){ // Message was found in the server. - count++; - } - } catch (ProcessingException | IllegalStateException e) {} // Read failed: try with next server - finally { - response.close(); - } - break; - - case READ_MESSAGES: - try { - msgList = response.readEntity(BulletinBoardMessageList.class); // If a BulletinBoardMessageList is returned: the read was successful - return new BulletinClientJobResult(job, msgList); // Return the result - } catch (ProcessingException | IllegalStateException e) {} // Read failed: try with next server - finally { - response.close(); - } - break; - - } - - } - - // Return result (if haven't done so yet) - switch(job.getJobType()) { - - case POST_MESSAGE: - // The job now contains the information required to ascertain whether enough server posts have succeeded - // It will also contain the list of servers in which the post was not successful - job.updateServerAddresses(serverAddresses); - return new BulletinClientJobResult(job, null); - - case GET_REDUNDANCY: - // Return the number of servers in which the message was found - // The job now contains the list of these servers - return new BulletinClientJobResult(job, IntMsg.newBuilder().setValue(count).build()); - - case READ_MESSAGES: - // A successful operation would have already returned an output - // Therefore: no server access was successful - throw new CommunicationException("Could not access any server"); - - default: // This is required for successful compilation - throw new IllegalArgumentException("Unsupported job type"); + public int getMaxRetry() { + return maxRetry; + } + public void decMaxRetry(){ + if (maxRetry > 0) { + maxRetry--; } } + + public boolean isRetry(){ + return (maxRetry != 0); + } + } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java new file mode 100644 index 0000000..7347f47 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java @@ -0,0 +1,104 @@ +package meerkat.bulletinboard; + +import com.google.common.util.concurrent.FutureCallback; + +import com.google.common.util.concurrent.FutureCallback; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by Arbel Deutsch Peled on 09-Dec-15. + * + * This is a general class for handling multi-server work + * It utilizes Single Server Clients to perform the actual per-server work + */ +public abstract class MultiServerWorker extends BulletinClientWorker implements Runnable, FutureCallback{ + + private final List clients; + + protected AtomicInteger minServers; // The minimal number of servers the job must be successful on for the job to be completed + + protected AtomicInteger maxFailedServers; // The maximal number of allowed server failures + + private AtomicBoolean returnedResult; + + private final FutureCallback futureCallback; + + /** + * Constructor + * @param clients contains a list of Single Server clients to handle requests + * @param shuffleClients is a boolean stating whether or not it is needed to shuffle the clients + * @param minServers is the minimal amount of servers needed in order to successfully complete the job + * @param payload is the payload for the job + * @param maxRetry is the maximal per-server retry count + * @param futureCallback contains the callback methods used to report the result back to the client + */ + public MultiServerWorker(List clients, boolean shuffleClients, + int minServers, IN payload, int maxRetry, + FutureCallback futureCallback) { + + super(payload,maxRetry); + + this.clients = clients; + if (shuffleClients){ + Collections.shuffle(clients); + } + + this.minServers = new AtomicInteger(minServers); + maxFailedServers = new AtomicInteger(clients.size() - minServers); + this.futureCallback = futureCallback; + + returnedResult = new AtomicBoolean(false); + + } + + /** + * Constructor overload without client shuffling + */ + public MultiServerWorker(List clients, + int minServers, IN payload, int maxRetry, + FutureCallback futureCallback) { + + this(clients, false, minServers, payload, maxRetry, futureCallback); + + } + + /** + * Used to report a successful operation to the client + * Only reports once to the client + * @param result is the result + */ + protected void succeed(OUT result){ + if (returnedResult.compareAndSet(false, true)) { + futureCallback.onSuccess(result); + } + } + + /** + * Used to report a failed operation to the client + * Only reports once to the client + * @param t contains the error/exception that occurred + */ + protected void fail(Throwable t){ + if (returnedResult.compareAndSet(false, true)) { + futureCallback.onFailure(t); + } + } + + /** + * Used by implementations to get a Single Server Client iterator + * @return the requested iterator + */ + protected Iterator getClientIterator() { + return clients.iterator(); + } + + protected int getClientNumber() { + return clients.size(); + } + +} \ No newline at end of file diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java index f340cae..633e495 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java @@ -5,8 +5,7 @@ import meerkat.comm.CommunicationException; import meerkat.crypto.Digest; import meerkat.crypto.concrete.SHA256Digest; import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.protobuf.Voting; -import meerkat.protobuf.Voting.BulletinBoardClientParams; +import meerkat.protobuf.Voting.*; import meerkat.rest.*; import java.util.List; @@ -17,25 +16,28 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; +import static meerkat.bulletinboard.BulletinBoardConstants.*; + /** * Created by Arbel Deutsch Peled on 05-Dec-15. + * Implements BulletinBoardClient interface in a simple, straightforward manner */ -public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { +public class SimpleBulletinBoardClient implements BulletinBoardClient{ - private List meerkatDBs; + protected List meerkatDBs; - private Client client; + protected Client client; - private Digest digest; + protected Digest digest; /** * Stores database locations and initializes the web Client * @param clientParams contains the data needed to access the DBs */ -// @Override - public void init(Voting.BulletinBoardClientParams clientParams) { + @Override + public void init(BulletinBoardClientParams clientParams) { - meerkatDBs = clientParams.getBulletinBoardAddressList(); + this.meerkatDBs = clientParams.getBulletinBoardAddressList(); client = ClientBuilder.newClient(); client.register(ProtobufMessageBodyReader.class); @@ -52,7 +54,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { * @return the message ID for later retrieval * @throws CommunicationException */ -// @Override + @Override public MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException { WebTarget webTarget; @@ -61,7 +63,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { // Post message to all databases try { for (String db : meerkatDBs) { - webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.POST_MESSAGE_PATH); + webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(POST_MESSAGE_PATH); response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF)); // Only consider valid responses @@ -88,7 +90,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { * @param id is the requested message ID * @return the number of DBs in which retrieval was successful */ -// @Override + @Override public float getRedundancy(MessageID id) { WebTarget webTarget; Response response; @@ -104,7 +106,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { for (String db : meerkatDBs) { try { - webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH); + webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF)); @@ -123,9 +125,9 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { * If at the operation is successful for some DB: return the results and stop iterating * If no operation is successful: return null (NOT blank list) * @param filterList return only messages that match the filters (null means no filtering). - * @return + * @return the list of Bulletin Board messages that are returned from a server */ -// @Override + @Override public List readMessages(MessageFilterList filterList) { WebTarget webTarget; Response response; @@ -138,7 +140,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { for (String db : meerkatDBs) { try { - webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH); + webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF)); @@ -154,8 +156,8 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient { return null; } -// @Override -// public void registerNewMessageCallback(MessageCallback callback, MessageFilterList filterList) { -// callback.handleNewMessage(readMessages(filterList)); -// } + public void close() { + client.close(); + } + } diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java new file mode 100644 index 0000000..ea0e93d --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java @@ -0,0 +1,609 @@ +package meerkat.bulletinboard; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.ByteString; +import meerkat.bulletinboard.workers.singleserver.*; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Voting.BulletinBoardClientParams; +import meerkat.util.BulletinBoardUtils; + +import javax.ws.rs.NotFoundException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by Arbel Deutsch Peled on 28-Dec-15. + * + * This class implements the asynchronous Bulletin Board Client interface + * It only handles a single Bulletin Board Server + * If the list of servers contains more than one server: the server actually used is the first one + * The class further implements a delayed access to the server after a communication error occurs + */ +public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient implements AsyncBulletinBoardClient { + + private final int MAX_RETRIES = 11; + + private ListeningScheduledExecutorService executorService; + + protected BatchDigest batchDigest; + + private long lastServerErrorTime; + + private final long failDelayInMilliseconds; + + private final long subscriptionIntervalInMilliseconds; + + /** + * Notify the client that a job has failed + * This makes new scheduled jobs be scheduled for a later time (after the given delay) + */ + protected void fail() { + + // Update last fail time + lastServerErrorTime = System.currentTimeMillis(); + + } + + /** + * This method adds a worker to the scheduled queue of the threadpool + * If the server is in an accessible state: the job is submitted for immediate handling + * If the server is not accessible: the job is scheduled for a later time + * @param worker is the worker that should be scheduled for work + * @param callback is the class containing callbacks for handling job completion/failure + */ + protected void scheduleWorker(SingleServerWorker worker, FutureCallback callback){ + + long timeSinceLastServerError = System.currentTimeMillis() - lastServerErrorTime; + + if (timeSinceLastServerError >= failDelayInMilliseconds) { + + // Schedule for immediate processing + Futures.addCallback(executorService.submit(worker), callback); + + } else { + + // Schedule for processing immediately following delay expiry + Futures.addCallback(executorService.schedule( + worker, + failDelayInMilliseconds - timeSinceLastServerError, + TimeUnit.MILLISECONDS), + callback); + + } + + } + + /** + * Inner class for handling simple operation results and retrying if needed + */ + class RetryCallback implements FutureCallback { + + private final SingleServerWorker worker; + private final FutureCallback futureCallback; + + public RetryCallback(SingleServerWorker worker, FutureCallback futureCallback) { + this.worker = worker; + this.futureCallback = futureCallback; + } + + @Override + public void onSuccess(T result) { + futureCallback.onSuccess(result); + } + + @Override + public void onFailure(Throwable t) { + + // Notify client about failure + fail(); + + // Check if another attempt should be made + + worker.decMaxRetry(); + + if (worker.isRetry()) { + // Perform another attempt + scheduleWorker(worker, this); + } else { + // No more retries: notify caller about failure + futureCallback.onFailure(t); + } + + } + + } + + /** + * This callback ties together all the per-batch-data callbacks into a single callback + * It reports success back to the user only if all of the batch-data were successfully posted + * If any batch-data fails to post: this callback reports failure + */ + class PostBatchDataListCallback implements FutureCallback { + + private final FutureCallback callback; + + private AtomicInteger batchDataRemaining; + private AtomicBoolean aggregatedResult; + + public PostBatchDataListCallback(int batchDataLength, FutureCallback callback) { + + this.callback = callback; + this.batchDataRemaining = new AtomicInteger(batchDataLength); + this.aggregatedResult = new AtomicBoolean(false); + + } + + @Override + public void onSuccess(Boolean result) { + + if (result){ + this.aggregatedResult.set(true); + } + + if (batchDataRemaining.decrementAndGet() == 0){ + callback.onSuccess(this.aggregatedResult.get()); + } + } + + @Override + public void onFailure(Throwable t) { + + // Notify caller about failure + callback.onFailure(t); + + } + } + + /** + * This callback ties together the different parts of a CompleteBatch as they arrive from the server + * It assembles a CompleteBatch from the parts and sends it to the user if all parts arrived + * If any part fails to arrive: it invokes the onFailure method + */ + class CompleteBatchReadCallback { + + private final FutureCallback callback; + + private List batchDataList; + private BulletinBoardMessage batchMessage; + + private AtomicInteger remainingQueries; + private AtomicBoolean failed; + + public CompleteBatchReadCallback(FutureCallback callback) { + + this.callback = callback; + + remainingQueries = new AtomicInteger(2); + failed = new AtomicBoolean(false); + + } + + protected void combineAndReturn() { + + final String[] prefixes = { + BulletinBoardConstants.BATCH_ID_TAG_PREFIX, + BulletinBoardConstants.BATCH_TAG}; + + if (remainingQueries.decrementAndGet() == 0){ + + String batchIdStr = BulletinBoardUtils.findTagWithPrefix(batchMessage, BulletinBoardConstants.BATCH_ID_TAG_PREFIX); + + if (batchIdStr == null){ + callback.onFailure(new CommunicationException("Server returned invalid message with no Batch ID tag")); + } + + BeginBatchMessage beginBatchMessage = + BeginBatchMessage.newBuilder() + .setSignerId(batchMessage.getSig(0).getSignerId()) + .setBatchId(Integer.parseInt(batchIdStr)) + .addAllTag(BulletinBoardUtils.removePrefixTags(batchMessage, Arrays.asList(prefixes))) + .build(); + callback.onSuccess(new CompleteBatch(beginBatchMessage, batchDataList, batchMessage.getSig(0))); + } + + } + + protected void fail(Throwable t) { + if (failed.compareAndSet(false, true)) { + callback.onFailure(t); + } + } + + /** + * @return a FutureCallback for the Batch Data List that ties to this object + */ + public FutureCallback> asBatchDataListFutureCallback() { + return new FutureCallback>() { + + @Override + public void onSuccess(List result) { + batchDataList = result; + + combineAndReturn(); + } + + @Override + public void onFailure(Throwable t) { + fail(t); + } + + }; + } + + /** + * @return a FutureCallback for the Bulletin Board Message that ties to this object + */ + public FutureCallback> asBulletinBoardMessageListFutureCallback() { + return new FutureCallback>() { + + @Override + public void onSuccess(List result) { + if (result.size() < 1){ + onFailure(new IllegalArgumentException("Server returned empty message list")); + return; + } + + batchMessage = result.get(0); + + combineAndReturn(); + } + + @Override + public void onFailure(Throwable t) { + fail(t); + } + }; + } + + } + + + /** + * Inner class for handling returned values of subscription operations + * This class's methods also ensure continued operation of the subscription + */ + class SubscriptionCallback implements FutureCallback> { + + private SingleServerReadMessagesWorker worker; + private final MessageHandler messageHandler; + + private MessageFilterList.Builder filterBuilder; + + public SubscriptionCallback(SingleServerReadMessagesWorker worker, MessageHandler messageHandler) { + this.worker = worker; + this.messageHandler = messageHandler; + filterBuilder = worker.getPayload().toBuilder(); + + } + + @Override + public void onSuccess(List result) { + + // Report new messages to user + messageHandler.handleNewMessages(result); + + // Remove last filter from list (MIN_ENTRY one) + filterBuilder.removeFilter(filterBuilder.getFilterCount() - 1); + + // Add updated MIN_ENTRY filter (entry number is successor of last received entry's number) + filterBuilder.addFilter(MessageFilter.newBuilder() + .setType(FilterType.MIN_ENTRY) + .setEntry(result.get(result.size() - 1).getEntryNum() + 1) + .build()); + + // Create new worker with updated task + worker = new SingleServerReadMessagesWorker(worker.serverAddress, filterBuilder.build(), 1); + + // Schedule the worker + scheduleWorker(worker, this); + + } + + @Override + public void onFailure(Throwable t) { + + // Notify client about failure + fail(); + + // Reschedule exact same task + scheduleWorker(worker, this); + } + } + + public SingleServerBulletinBoardClient(int threadPoolSize, long failDelayInMilliseconds, long subscriptionIntervalInMilliseconds) { + + executorService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(threadPoolSize)); + + this.failDelayInMilliseconds = failDelayInMilliseconds; + this.subscriptionIntervalInMilliseconds = subscriptionIntervalInMilliseconds; + + // Set server error time to a time sufficiently in the past to make new jobs go through + lastServerErrorTime = System.currentTimeMillis() - failDelayInMilliseconds; + + } + + /** + * Stores database location, initializes the web Client and + * @param clientParams contains the data needed to access the DBs + */ + @Override + public void init(BulletinBoardClientParams clientParams) { + + // Perform usual setup + super.init(clientParams); + + // Wrap the Digest into a BatchDigest + batchDigest = new GenericBatchDigest(digest); + + // Remove all but first DB address + String dbAddress = meerkatDBs.get(0); + meerkatDBs = new LinkedList<>(); + meerkatDBs.add(dbAddress); + + } + + @Override + public MessageID postMessage(BulletinBoardMessage msg, FutureCallback callback) { + + // Create worker with redundancy 1 and MAX_RETRIES retries + SingleServerPostMessageWorker worker = new SingleServerPostMessageWorker(meerkatDBs.get(0), msg, MAX_RETRIES); + + // Submit worker and create callback + scheduleWorker(worker, new RetryCallback<>(worker, callback)); + + // Calculate the correct message ID and return it + batchDigest.reset(); + batchDigest.update(msg.getMsg()); + return batchDigest.digestAsMessageID(); + + } + + private class PostBatchDataCallback implements FutureCallback { + + private final CompleteBatch completeBatch; + private final FutureCallback callback; + + public PostBatchDataCallback(CompleteBatch completeBatch, FutureCallback callback) { + this.completeBatch = completeBatch; + this.callback = callback; + } + + @Override + public void onSuccess(Boolean msg) { + closeBatch( + CloseBatchMessage.newBuilder() + .setBatchId(completeBatch.getBeginBatchMessage().getBatchId()) + .setSig(completeBatch.getSignature()) + .setBatchLength(completeBatch.getBatchDataList().size()) + .build(), + callback + ); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + + } + + private class BeginBatchCallback implements FutureCallback { + + private final CompleteBatch completeBatch; + private final FutureCallback callback; + + public BeginBatchCallback(CompleteBatch completeBatch, FutureCallback callback) { + this.completeBatch = completeBatch; + this.callback = callback; + } + + @Override + public void onSuccess(Boolean msg) { + + postBatchData( + completeBatch.getBeginBatchMessage().getSignerId(), + completeBatch.getBeginBatchMessage().getBatchId(), + completeBatch.getBatchDataList(), + 0, + new PostBatchDataCallback(completeBatch,callback)); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + + @Override + public MessageID postBatch(CompleteBatch completeBatch, FutureCallback callback) { + + beginBatch( + completeBatch.getBeginBatchMessage(), + new BeginBatchCallback(completeBatch, callback) + ); + + batchDigest.update(completeBatch); + + return batchDigest.digestAsMessageID(); + + } + + @Override + public void beginBatch(BeginBatchMessage beginBatchMessage, FutureCallback callback) { + + // Create worker with redundancy 1 and MAX_RETRIES retries + SingleServerBeginBatchWorker worker = + new SingleServerBeginBatchWorker(meerkatDBs.get(0), beginBatchMessage, MAX_RETRIES); + + // Submit worker and create callback + scheduleWorker(worker, new RetryCallback<>(worker, callback)); + + } + + @Override + public void postBatchData(ByteString signerId, int batchId, List batchDataList, + int startPosition, FutureCallback callback) { + + BatchMessage.Builder builder = BatchMessage.newBuilder() + .setSignerId(signerId) + .setBatchId(batchId); + + // Create a unified callback to aggregate successful posts + + PostBatchDataListCallback listCallback = new PostBatchDataListCallback(batchDataList.size(), callback); + + // Iterate through data list + + for (BatchData data : batchDataList) { + builder.setSerialNum(startPosition).setData(data); + + // Create worker with redundancy 1 and MAX_RETRIES retries + SingleServerPostBatchWorker worker = + new SingleServerPostBatchWorker(meerkatDBs.get(0), builder.build(), MAX_RETRIES); + + // Create worker with redundancy 1 and MAX_RETRIES retries + scheduleWorker(worker, new RetryCallback<>(worker, listCallback)); + + // Increment position in batch + startPosition++; + } + + } + + @Override + public void postBatchData(ByteString signerId, int batchId, List batchDataList, FutureCallback callback) { + + postBatchData(signerId, batchId, batchDataList, 0, callback); + + } + + @Override + public void postBatchData(byte[] signerId, int batchId, List batchDataList, + int startPosition, FutureCallback callback) { + + postBatchData(ByteString.copyFrom(signerId), batchId, batchDataList, startPosition, callback); + + } + + @Override + public void postBatchData(byte[] signerId, int batchId, List batchDataList, FutureCallback callback) { + + postBatchData(signerId, batchId, batchDataList, 0, callback); + + } + + @Override + public void closeBatch(CloseBatchMessage closeBatchMessage, FutureCallback callback) { + + // Create worker with redundancy 1 and MAX_RETRIES retries + SingleServerCloseBatchWorker worker = + new SingleServerCloseBatchWorker(meerkatDBs.get(0), closeBatchMessage, MAX_RETRIES); + + // Submit worker and create callback + scheduleWorker(worker, new RetryCallback<>(worker, callback)); + + } + + @Override + public void getRedundancy(MessageID id, FutureCallback callback) { + + // Create worker with no retries + SingleServerGetRedundancyWorker worker = new SingleServerGetRedundancyWorker(meerkatDBs.get(0), id, 1); + + // Submit job and create callback + scheduleWorker(worker, new RetryCallback<>(worker, callback)); + + } + + @Override + public void readMessages(MessageFilterList filterList, FutureCallback> callback) { + + // Create job with no retries + SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterList, 1); + + // Submit job and create callback + scheduleWorker(worker, new RetryCallback<>(worker, callback)); + + } + + @Override + public void readBatch(BatchSpecificationMessage batchSpecificationMessage, FutureCallback callback) { + + // Create job with no retries for retrieval of the Bulletin Board Message that defines the batch + + MessageFilterList filterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(BulletinBoardConstants.BATCH_TAG) + .build()) + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(BulletinBoardConstants.BATCH_ID_TAG_PREFIX + batchSpecificationMessage.getBatchId()) + .build()) + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.SIGNER_ID) + .setId(batchSpecificationMessage.getSignerId()) + .build()) + .build(); + + SingleServerReadMessagesWorker messageWorker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterList, 1); + + // Create job with no retries for retrieval of the Batch Data List + SingleServerReadBatchWorker batchWorker = new SingleServerReadBatchWorker(meerkatDBs.get(0), batchSpecificationMessage, 1); + + // Create callback that will combine the two worker products + CompleteBatchReadCallback completeBatchReadCallback = new CompleteBatchReadCallback(callback); + + // Submit jobs with wrapped callbacks + scheduleWorker(messageWorker, new RetryCallback<>(messageWorker, completeBatchReadCallback.asBulletinBoardMessageListFutureCallback())); + scheduleWorker(batchWorker, new RetryCallback<>(batchWorker, completeBatchReadCallback.asBatchDataListFutureCallback())); + + } + + @Override + public void subscribe(MessageFilterList filterList, MessageHandler messageHandler) { + + // Remove all existing MIN_ENTRY filters and create new one that starts at 0 + + MessageFilterList.Builder filterListBuilder = filterList.toBuilder(); + + Iterator iterator = filterListBuilder.getFilterList().iterator(); + while (iterator.hasNext()) { + MessageFilter filter = iterator.next(); + + if (filter.getType() == FilterType.MIN_ENTRY){ + iterator.remove(); + } + } + filterListBuilder.addFilter(MessageFilter.newBuilder() + .setType(FilterType.MIN_ENTRY) + .setEntry(0) + .build()); + + // Create job with no retries + SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterListBuilder.build(), 1); + + // Submit job and create callback + scheduleWorker(worker, new SubscriptionCallback(worker, messageHandler)); + + } + + @Override + public void close() { + + super.close(); + + executorService.shutdown(); + + } +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java new file mode 100644 index 0000000..ecebc12 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java @@ -0,0 +1,39 @@ +package meerkat.bulletinboard; + +import meerkat.rest.ProtobufMessageBodyReader; +import meerkat.rest.ProtobufMessageBodyWriter; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import java.util.concurrent.Callable; + +/** + * Created by Arbel Deutsch Peled on 02-Jan-16. + */ +public abstract class SingleServerWorker extends BulletinClientWorker implements Callable{ + + // This resource enabled creation of a single Client per thread. + protected static final ThreadLocal clientLocal = + new ThreadLocal () { + @Override protected Client initialValue() { + Client client; + client = ClientBuilder.newClient(); + client.register(ProtobufMessageBodyReader.class); + client.register(ProtobufMessageBodyWriter.class); + + return client; + } + }; + + protected final String serverAddress; + + public SingleServerWorker(String serverAddress, IN payload, int maxRetry) { + super(payload, maxRetry); + this.serverAddress = serverAddress; + } + + public String getServerAddress() { + return serverAddress; + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java index bb46c32..d8f66f2 100644 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java @@ -1,39 +1,44 @@ package meerkat.bulletinboard; -import com.google.common.util.concurrent.*; +import com.google.common.util.concurrent.FutureCallback; import com.google.protobuf.ByteString; -import meerkat.bulletinboard.callbacks.GetRedundancyFutureCallback; -import meerkat.bulletinboard.callbacks.PostMessageFutureCallback; -import meerkat.bulletinboard.callbacks.ReadMessagesFutureCallback; -import meerkat.comm.CommunicationException; -import meerkat.crypto.Digest; -import meerkat.crypto.concrete.SHA256Digest; -import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.protobuf.Voting; +import meerkat.bulletinboard.workers.multiserver.*; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Voting.*; + +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; /** * Created by Arbel Deutsch Peled on 05-Dec-15. - * Thread-based implementation of a Bulletin Board Client. + * Thread-based implementation of a Async Bulletin Board Client. * Features: * 1. Handles tasks concurrently. * 2. Retries submitting */ -public class ThreadedBulletinBoardClient implements BulletinBoardClient { +public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient implements AsyncBulletinBoardClient { - private final static int THREAD_NUM = 10; - ListeningExecutorService listeningExecutor; + // Executor service for handling jobs + private final static int JOBS_THREAD_NUM = 5; + private ExecutorService executorService; - private Digest digest; + // Per-server clients + private List clients; - private List meerkatDBs; - private String postSubAddress; - private String readSubAddress; + private BatchDigest batchDigest; + private final static int POST_MESSAGE_RETRY_NUM = 3; private final static int READ_MESSAGES_RETRY_NUM = 1; + private final static int GET_REDUNDANCY_RETRY_NUM = 1; + + private static final int SERVER_THREADPOOL_SIZE = 5; + private static final long FAIL_DELAY = 5000; + private static final long SUBSCRIPTION_INTERVAL = 10000; private int minAbsoluteRedundancy; @@ -44,15 +49,29 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient { * @param clientParams contains the required information */ @Override - public void init(Voting.BulletinBoardClientParams clientParams) { + public void init(BulletinBoardClientParams clientParams) { - meerkatDBs = clientParams.getBulletinBoardAddressList(); + super.init(clientParams); - minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * meerkatDBs.size()); + batchDigest = new GenericBatchDigest(digest); - listeningExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(THREAD_NUM)); + minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * clientParams.getBulletinBoardAddressCount()); - digest = new SHA256Digest(); + executorService = Executors.newFixedThreadPool(JOBS_THREAD_NUM); + + clients = new ArrayList<>(clientParams.getBulletinBoardAddressCount()); + for (String address : clientParams.getBulletinBoardAddressList()){ + + SingleServerBulletinBoardClient client = + new SingleServerBulletinBoardClient(SERVER_THREADPOOL_SIZE, FAIL_DELAY, SUBSCRIPTION_INTERVAL); + + client.init(BulletinBoardClientParams.newBuilder() + .addBulletinBoardAddress(address) + .build()); + + clients.add(client); + + } } @@ -61,21 +80,100 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient { * Retry failed DBs * @param msg is the message, * @return the message ID for later retrieval - * @throws CommunicationException */ @Override - public MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback){ + public MessageID postMessage(BulletinBoardMessage msg, FutureCallback callback){ // Create job - BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.POST_MESSAGE, msg, -1); + MultiServerPostMessageWorker worker = + new MultiServerPostMessageWorker(clients, minAbsoluteRedundancy, msg, POST_MESSAGE_RETRY_NUM, callback); - // Submit job and create callback - Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new PostMessageFutureCallback(listeningExecutor, callback)); + // Submit job + executorService.submit(worker); // Calculate the correct message ID and return it - digest.reset(); - digest.update(msg.getMsg()); - return MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build(); + batchDigest.reset(); + batchDigest.update(msg.getMsg()); + return batchDigest.digestAsMessageID(); + + } + + @Override + public MessageID postBatch(CompleteBatch completeBatch, FutureCallback callback) { + + // Create job + MultiServerPostBatchWorker worker = + new MultiServerPostBatchWorker(clients, minAbsoluteRedundancy, completeBatch, POST_MESSAGE_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); + + // Calculate the correct message ID and return it + batchDigest.reset(); + batchDigest.update(completeBatch); + return batchDigest.digestAsMessageID(); + + } + + @Override + public void beginBatch(BeginBatchMessage beginBatchMessage, FutureCallback callback) { + + // Create job + MultiServerBeginBatchWorker worker = + new MultiServerBeginBatchWorker(clients, minAbsoluteRedundancy, beginBatchMessage, POST_MESSAGE_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); + + } + + @Override + public void postBatchData(byte[] signerId, int batchId, List batchDataList, + int startPosition, FutureCallback callback) { + + BatchDataContainer batchDataContainer = new BatchDataContainer(signerId, batchId, batchDataList, startPosition); + + // Create job + MultiServerPostBatchDataWorker worker = + new MultiServerPostBatchDataWorker(clients, minAbsoluteRedundancy, batchDataContainer, POST_MESSAGE_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); + + } + + @Override + public void postBatchData(byte[] signerId, int batchId, List batchDataList, FutureCallback callback) { + + postBatchData(signerId, batchId, batchDataList, 0, callback); + + } + + @Override + public void postBatchData(ByteString signerId, int batchId, List batchDataList, + int startPosition, FutureCallback callback) { + + postBatchData(signerId.toByteArray(), batchId, batchDataList, startPosition, callback); + + } + + @Override + public void postBatchData(ByteString signerId, int batchId, List batchDataList, FutureCallback callback) { + + postBatchData(signerId, batchId, batchDataList, 0, callback); + + } + + @Override + public void closeBatch(CloseBatchMessage closeBatchMessage, FutureCallback callback) { + + // Create job + MultiServerCloseBatchWorker worker = + new MultiServerCloseBatchWorker(clients, minAbsoluteRedundancy, closeBatchMessage, POST_MESSAGE_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); + } /** @@ -83,17 +181,16 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient { * Return the number of databases in which the message was found * Only try once per DB * Ignore communication exceptions in specific databases - * @param id is the requested message ID - * @return the number of DBs in which retrieval was successful */ @Override - public void getRedundancy(MessageID id, ClientCallback callback) { + public void getRedundancy(MessageID id, FutureCallback callback) { // Create job - BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.GET_REDUNDANCY, id, 1); + MultiServerGetRedundancyWorker worker = + new MultiServerGetRedundancyWorker(clients, minAbsoluteRedundancy, id, GET_REDUNDANCY_RETRY_NUM, callback); - // Submit job and create callback - Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new GetRedundancyFutureCallback(listeningExecutor, callback)); + // Submit job + executorService.submit(worker); } @@ -101,27 +198,49 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient { * Go through the DBs and try to retrieve messages according to the specified filter * If at the operation is successful for some DB: return the results and stop iterating * If no operation is successful: return null (NOT blank list) - * @param filterList return only messages that match the filters (null means no filtering). - * @return */ @Override - public void readMessages(MessageFilterList filterList, ClientCallback> callback) { + public void readMessages(MessageFilterList filterList, FutureCallback> callback) { // Create job - BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.READ_MESSAGES, - filterList, READ_MESSAGES_RETRY_NUM); + MultiServerReadMessagesWorker worker = + new MultiServerReadMessagesWorker(clients, minAbsoluteRedundancy, filterList, READ_MESSAGES_RETRY_NUM, callback); - // Submit job and create callback - Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new ReadMessagesFutureCallback(listeningExecutor, callback)); + // Submit job + executorService.submit(worker); } + @Override + public void readBatch(BatchSpecificationMessage batchSpecificationMessage, FutureCallback callback) { + + // Create job + MultiServerReadBatchWorker worker = + new MultiServerReadBatchWorker(clients, minAbsoluteRedundancy, batchSpecificationMessage, READ_MESSAGES_RETRY_NUM, callback); + + // Submit job + executorService.submit(worker); + + } + + @Override + public void subscribe(MessageFilterList filterList, MessageHandler messageHandler) { + // TODO: Implement + } + @Override public void close() { + super.close(); + try { - listeningExecutor.shutdown(); - while (! listeningExecutor.isShutdown()) { - listeningExecutor.awaitTermination(10, TimeUnit.SECONDS); + + for (SingleServerBulletinBoardClient client : clients){ + client.close(); + } + + executorService.shutdown(); + while (! executorService.isShutdown()) { + executorService.awaitTermination(10, TimeUnit.SECONDS); } } catch (InterruptedException e) { System.err.println(e.getCause() + " " + e.getMessage()); diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java deleted file mode 100644 index 7c5b7b0..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java +++ /dev/null @@ -1,25 +0,0 @@ -package meerkat.bulletinboard.callbacks; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.BulletinClientJob; -import meerkat.bulletinboard.BulletinClientJobResult; -import meerkat.bulletinboard.BulletinClientWorker; -import meerkat.protobuf.BulletinBoardAPI; - -import java.util.List; - -/** - * This is a future callback used to listen to workers and run on job finish - * Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error - */ -public abstract class ClientFutureCallback implements FutureCallback { - - protected ListeningExecutorService listeningExecutor; - - ClientFutureCallback(ListeningExecutorService listeningExecutor) { - this.listeningExecutor = listeningExecutor; - } - -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java deleted file mode 100644 index 518ed77..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java +++ /dev/null @@ -1,38 +0,0 @@ -package meerkat.bulletinboard.callbacks; - -import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.BulletinBoardClient; -import meerkat.bulletinboard.BulletinClientJobResult; -import meerkat.protobuf.BulletinBoardAPI.*; - -import java.util.List; - -/** - * This is a future callback used to listen to workers and run on job finish - * Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error - */ -public class GetRedundancyFutureCallback extends ClientFutureCallback { - - private BulletinBoardClient.ClientCallback callback; - - public GetRedundancyFutureCallback(ListeningExecutorService listeningExecutor, - BulletinBoardClient.ClientCallback callback) { - super(listeningExecutor); - this.callback = callback; - } - - @Override - public void onSuccess(BulletinClientJobResult result) { - - int absoluteRedundancy = ((IntMsg) result.getResult()).getValue(); - int totalServers = result.getJob().getServerAddresses().size(); - - callback.handleCallback( ((float) absoluteRedundancy) / ((float) totalServers) ); - - } - - @Override - public void onFailure(Throwable t) { - callback.handleFailure(t); - } -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java deleted file mode 100644 index 221ae1a..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java +++ /dev/null @@ -1,46 +0,0 @@ -package meerkat.bulletinboard.callbacks; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.BulletinBoardClient; -import meerkat.bulletinboard.BulletinClientJob; -import meerkat.bulletinboard.BulletinClientJobResult; -import meerkat.bulletinboard.BulletinClientWorker; -import meerkat.protobuf.BulletinBoardAPI; - -import java.util.List; - -/** - * This is a future callback used to listen to workers and run on job finish - * Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error - */ -public class PostMessageFutureCallback extends ClientFutureCallback { - - private BulletinBoardClient.ClientCallback callback; - - public PostMessageFutureCallback(ListeningExecutorService listeningExecutor, - BulletinBoardClient.ClientCallback callback) { - super(listeningExecutor); - this.callback = callback; - } - - @Override - public void onSuccess(BulletinClientJobResult result) { - - BulletinClientJob job = result.getJob(); - - job.decMaxRetry(); - - // If redundancy is below threshold: retry - if (job.getMinServers() > 0 && job.isRetry()) { - Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), this); - } - - callback.handleCallback(null); - } - - @Override - public void onFailure(Throwable t) { - callback.handleFailure(t); - } -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java deleted file mode 100644 index 4c43ba2..0000000 --- a/bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java +++ /dev/null @@ -1,38 +0,0 @@ -package meerkat.bulletinboard.callbacks; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListeningExecutorService; -import meerkat.bulletinboard.BulletinBoardClient; -import meerkat.bulletinboard.BulletinClientJob; -import meerkat.bulletinboard.BulletinClientJobResult; -import meerkat.bulletinboard.BulletinClientWorker; -import meerkat.protobuf.BulletinBoardAPI; - -import java.util.List; - -/** - * This is a future callback used to listen to workers and run on job finish - * Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error - */ -public class ReadMessagesFutureCallback extends ClientFutureCallback { - - private BulletinBoardClient.ClientCallback> callback; - - public ReadMessagesFutureCallback(ListeningExecutorService listeningExecutor, - BulletinBoardClient.ClientCallback> callback) { - super(listeningExecutor); - this.callback = callback; - } - - @Override - public void onSuccess(BulletinClientJobResult result) { - - callback.handleCallback(((BulletinBoardAPI.BulletinBoardMessageList) result.getResult()).getMessageList()); - - } - - @Override - public void onFailure(Throwable t) { - callback.handleFailure(t); - } -} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java new file mode 100644 index 0000000..e0e92bb --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerBeginBatchWorker extends MultiServerGenericPostWorker { + + public MultiServerBeginBatchWorker(List clients, + int minServers, BeginBatchMessage payload, int maxRetry, + FutureCallback futureCallback) { + + super(clients, minServers, payload, maxRetry, futureCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, BeginBatchMessage payload) { + client.beginBatch(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java new file mode 100644 index 0000000..300440f --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.CloseBatchMessage; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerCloseBatchWorker extends MultiServerGenericPostWorker { + + public MultiServerCloseBatchWorker(List clients, + int minServers, CloseBatchMessage payload, int maxRetry, + FutureCallback futureCallback) { + + super(clients, minServers, payload, maxRetry, futureCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, CloseBatchMessage payload) { + client.closeBatch(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java new file mode 100644 index 0000000..8172e14 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java @@ -0,0 +1,67 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.MultiServerWorker; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.comm.CommunicationException; + +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; +import java.util.Iterator; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public abstract class MultiServerGenericPostWorker extends MultiServerWorker { + + public MultiServerGenericPostWorker(List clients, + int minServers, T payload, int maxRetry, + FutureCallback futureCallback) { + + super(clients, minServers, payload, maxRetry, futureCallback); + + } + + protected abstract void doPost(SingleServerBulletinBoardClient client, T payload); + + /** + * This method carries out the actual communication with the servers via HTTP Post + * It accesses the servers one by one and tries to post the payload to each in turn + * The method will only iterate once through the server list + * Successful post to a server results in removing the server from the list + */ + public void run() { + + // Iterate through servers + + Iterator clientIterator = getClientIterator(); + + while (clientIterator.hasNext()) { + + // Send request to Server + SingleServerBulletinBoardClient client = clientIterator.next(); + + doPost(client, payload); + + } + + } + + @Override + public void onSuccess(Boolean result) { + if (result){ + if (minServers.decrementAndGet() <= 0){ + succeed(Boolean.TRUE); + } + } + } + + @Override + public void onFailure(Throwable t) { + if (maxFailedServers.decrementAndGet() < 0){ + fail(t); + } + } +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java new file mode 100644 index 0000000..88b4ac1 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java @@ -0,0 +1,64 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.MultiServerWorker; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.comm.CommunicationException; + +import java.util.Iterator; +import java.util.List; + + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public abstract class MultiServerGenericReadWorker extends MultiServerWorker{ + + private final Iterator clientIterator; + + public MultiServerGenericReadWorker(List clients, + int minServers, IN payload, int maxRetry, + FutureCallback futureCallback) { + + super(clients, true, minServers, payload, maxRetry, futureCallback); // Shuffle clients on creation to balance load + + clientIterator = getClientIterator(); + + } + + protected abstract void doRead(IN payload, SingleServerBulletinBoardClient client); + + /** + * This method carries out the actual communication with the servers via HTTP Post + * It accesses the servers in a random order until one answers it + * Successful retrieval from any server terminates the method and returns the received values; The list is not changed + */ + public void run(){ + + // Iterate through servers + + if (clientIterator.hasNext()) { + + // Get next server + SingleServerBulletinBoardClient client = clientIterator.next(); + + // Retrieve answer from server + doRead(payload, client); + + } else { + fail(new CommunicationException("Could not contact any server")); + } + + } + + @Override + public void onSuccess(OUT msg) { + succeed(msg); + } + + @Override + public void onFailure(Throwable t) { + run(); // Retry with next server + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java new file mode 100644 index 0000000..748916b --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java @@ -0,0 +1,72 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.MultiServerWorker; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerGetRedundancyWorker extends MultiServerWorker { + + private AtomicInteger serversContainingMessage; + private AtomicInteger totalContactedServers; + + public MultiServerGetRedundancyWorker(List clients, + int minServers, MessageID payload, int maxRetry, + FutureCallback futureCallback) { + + super(clients, minServers, payload, maxRetry, futureCallback); // Shuffle clients on creation to balance load + + serversContainingMessage = new AtomicInteger(0); + totalContactedServers = new AtomicInteger(0); + + } + + /** + * This method carries out the actual communication with the servers via HTTP Post + * It accesses the servers in a random order until one answers it + * Successful retrieval from any server terminates the method and returns the received values; The list is not changed + */ + public void run(){ + + Iterator clientIterator = getClientIterator(); + + // Iterate through clients + + while (clientIterator.hasNext()) { + + SingleServerBulletinBoardClient client = clientIterator.next(); + + // Send request to client + client.getRedundancy(payload,this); + + } + + } + + @Override + public void onSuccess(Float result) { + + if (result > 0.5) { + serversContainingMessage.incrementAndGet(); + } + + if (totalContactedServers.incrementAndGet() >= getClientNumber()){ + succeed(((float) serversContainingMessage.get()) / ((float) getClientNumber())); + } + + } + + @Override + public void onFailure(Throwable t) { + onSuccess(0.0f); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchDataWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchDataWorker.java new file mode 100644 index 0000000..ef1c4fa --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchDataWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.bulletinboard.BatchDataContainer; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerPostBatchDataWorker extends MultiServerGenericPostWorker { + + public MultiServerPostBatchDataWorker(List clients, + int minServers, BatchDataContainer payload, int maxRetry, + FutureCallback futureCallback) { + + super(clients, minServers, payload, maxRetry, futureCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, BatchDataContainer payload) { + client.postBatchData(payload.signerId, payload.batchId, payload.batchDataList, payload.startPosition, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java new file mode 100644 index 0000000..1b1f3df --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.CompleteBatch; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerPostBatchWorker extends MultiServerGenericPostWorker { + + public MultiServerPostBatchWorker(List clients, + int minServers, CompleteBatch payload, int maxRetry, + FutureCallback futureCallback) { + + super(clients, minServers, payload, maxRetry, futureCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, CompleteBatch payload) { + client.postBatch(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java new file mode 100644 index 0000000..6d3d702 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java @@ -0,0 +1,28 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerPostMessageWorker extends MultiServerGenericPostWorker { + + public MultiServerPostMessageWorker(List clients, + int minServers, BulletinBoardMessage payload, int maxRetry, + FutureCallback futureCallback) { + + super(clients, minServers, payload, maxRetry, futureCallback); + + } + + @Override + protected void doPost(SingleServerBulletinBoardClient client, BulletinBoardMessage payload) { + client.postMessage(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java new file mode 100644 index 0000000..3d40c8a --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java @@ -0,0 +1,30 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.CompleteBatch; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.BatchSpecificationMessage; + +import java.util.List; + + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerReadBatchWorker extends MultiServerGenericReadWorker { + + public MultiServerReadBatchWorker(List clients, + int minServers, BatchSpecificationMessage payload, int maxRetry, + FutureCallback futureCallback) { + + super(clients, minServers, payload, maxRetry, futureCallback); + + } + + @Override + protected void doRead(BatchSpecificationMessage payload, SingleServerBulletinBoardClient client) { + client.readBatch(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java new file mode 100644 index 0000000..980d869 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java @@ -0,0 +1,29 @@ +package meerkat.bulletinboard.workers.multiserver; + +import com.google.common.util.concurrent.FutureCallback; +import meerkat.bulletinboard.SingleServerBulletinBoardClient; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class MultiServerReadMessagesWorker extends MultiServerGenericReadWorker>{ + + public MultiServerReadMessagesWorker(List clients, + int minServers, MessageFilterList payload, int maxRetry, + FutureCallback> futureCallback) { + + super(clients, minServers, payload, maxRetry, futureCallback); + + } + + @Override + protected void doRead(MessageFilterList payload, SingleServerBulletinBoardClient client) { + client.readMessages(payload, this); + } + + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerBeginBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerBeginBatchWorker.java new file mode 100644 index 0000000..0c1a1a3 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerBeginBatchWorker.java @@ -0,0 +1,17 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; + +import static meerkat.bulletinboard.BulletinBoardConstants.BEGIN_BATCH_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + * Tries to contact server once and perform a post operation + */ +public class SingleServerBeginBatchWorker extends SingleServerGenericPostWorker { + + public SingleServerBeginBatchWorker(String serverAddress, BeginBatchMessage payload, int maxRetry) { + super(serverAddress, BEGIN_BATCH_PATH, payload, maxRetry); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerCloseBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerCloseBatchWorker.java new file mode 100644 index 0000000..ab298a5 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerCloseBatchWorker.java @@ -0,0 +1,17 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.protobuf.BulletinBoardAPI.CloseBatchMessage; + +import static meerkat.bulletinboard.BulletinBoardConstants.CLOSE_BATCH_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + * Tries to contact server once and perform a close batch operation + */ +public class SingleServerCloseBatchWorker extends SingleServerGenericPostWorker { + + public SingleServerCloseBatchWorker(String serverAddress, CloseBatchMessage payload, int maxRetry) { + super(serverAddress, CLOSE_BATCH_PATH, payload, maxRetry); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java new file mode 100644 index 0000000..621a828 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java @@ -0,0 +1,62 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.bulletinboard.SingleServerWorker; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.BoolMsg; +import meerkat.rest.Constants; + +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; + +import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + * Tries to contact server once and perform a post operation + */ +public class SingleServerGenericPostWorker extends SingleServerWorker { + + private final String subPath; + + public SingleServerGenericPostWorker(String serverAddress, String subPath, T payload, int maxRetry) { + super(serverAddress, payload, maxRetry); + this.subPath = subPath; + } + + /** + * This method carries out the actual communication with the server via HTTP Post + * It accesses the server and tries to post the payload to it + * Successful post to a server results + * @return TRUE if the operation is successful + * @throws CommunicationException if the operation is unsuccessful + */ + public Boolean call() throws CommunicationException{ + + Client client = clientLocal.get(); + + WebTarget webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(subPath); + Response response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post( + Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF)); + + try { + + // If a BoolMsg entity is returned: the post was successful + response.readEntity(BoolMsg.class); + return Boolean.TRUE; + + } catch (ProcessingException | IllegalStateException e) { + + // Post to this server failed + throw new CommunicationException("Could not contact the server"); + + } + finally { + response.close(); + } + + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGetRedundancyWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGetRedundancyWorker.java new file mode 100644 index 0000000..10517f7 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGetRedundancyWorker.java @@ -0,0 +1,79 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.bulletinboard.SingleServerWorker; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.rest.Constants; + +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; + +import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; +import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class SingleServerGetRedundancyWorker extends SingleServerWorker { + + public SingleServerGetRedundancyWorker(String serverAddress, MessageID payload, int maxRetry) { + super(serverAddress, payload, maxRetry); + } + + /** + * This method carries out the actual communication with the server via HTTP Post + * It queries the server for a message with the given ID + * @return TRUE if the message exists in the server and FALSE otherwise + * @throws CommunicationException if the server does not return a valid answer + */ + public Float call() throws CommunicationException{ + + Client client = clientLocal.get(); + + WebTarget webTarget; + Response response; + + MessageFilterList msgFilterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.MSG_ID) + .setId(payload.getID()) + .build() + ).build(); + + // Send request to Server + + webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); + response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msgFilterList, Constants.MEDIATYPE_PROTOBUF)); + + // Retrieve answer + + try { + + // If a BulletinBoardMessageList is returned: the read was successful + BulletinBoardMessageList msgList = response.readEntity(BulletinBoardMessageList.class); + + if (msgList.getMessageList().size() > 0){ + // Message exists in the server + return 1.0f; + } + else { + // Message does not exist in the server + return 0.0f; + } + + } catch (ProcessingException | IllegalStateException e) { + + // Read failed + throw new CommunicationException("Server access failed"); + + } + finally { + response.close(); + } + + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostBatchWorker.java new file mode 100644 index 0000000..dfb42a7 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostBatchWorker.java @@ -0,0 +1,17 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.protobuf.BulletinBoardAPI.BatchMessage; + +import static meerkat.bulletinboard.BulletinBoardConstants.POST_BATCH_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + * Tries to contact server once and perform a post batch operation + */ +public class SingleServerPostBatchWorker extends SingleServerGenericPostWorker { + + public SingleServerPostBatchWorker(String serverAddress, BatchMessage payload, int maxRetry) { + super(serverAddress, POST_BATCH_PATH, payload, maxRetry); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostMessageWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostMessageWorker.java new file mode 100644 index 0000000..454d720 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostMessageWorker.java @@ -0,0 +1,17 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; + +import static meerkat.bulletinboard.BulletinBoardConstants.POST_MESSAGE_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + * Tries to contact server once and perform a post operation + */ +public class SingleServerPostMessageWorker extends SingleServerGenericPostWorker { + + public SingleServerPostMessageWorker(String serverAddress, BulletinBoardMessage payload, int maxRetry) { + super(serverAddress, POST_MESSAGE_PATH, payload, maxRetry); + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java new file mode 100644 index 0000000..11fc777 --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java @@ -0,0 +1,70 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.bulletinboard.CompleteBatch; +import meerkat.bulletinboard.SingleServerWorker; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.rest.Constants; + +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.GenericType; +import javax.ws.rs.core.Response; +import java.util.List; + +import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; +import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH; +import static meerkat.bulletinboard.BulletinBoardConstants.READ_BATCH_PATH; + +import static meerkat.bulletinboard.BulletinBoardConstants.BATCH_ID_TAG_PREFIX; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class SingleServerReadBatchWorker extends SingleServerWorker> { + + public SingleServerReadBatchWorker(String serverAddress, BatchSpecificationMessage payload, int maxRetry) { + super(serverAddress, payload, maxRetry); + } + + /** + * This method carries out the actual communication with the server via HTTP Post + * Upon successful retrieval from the server the method returns the received values + * @return the complete batch as read from the server + * @throws CommunicationException if the server's response is invalid + */ + public List call() throws CommunicationException{ + + Client client = clientLocal.get(); + + WebTarget webTarget; + Response response; + + // Get the batch data + + webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_BATCH_PATH); + response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post( + Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF)); + + // Retrieve answer + + try { + + // If a BatchDataList is returned: the read was successful + return response.readEntity(BatchDataList.class).getDataList(); + + } catch (ProcessingException | IllegalStateException e) { + + // Read failed + throw new CommunicationException("Could not contact the server"); + + } + finally { + response.close(); + } + + } + +} diff --git a/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java new file mode 100644 index 0000000..6c09bcc --- /dev/null +++ b/bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java @@ -0,0 +1,68 @@ +package meerkat.bulletinboard.workers.singleserver; + +import meerkat.bulletinboard.SingleServerWorker; +import meerkat.comm.CommunicationException; +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessageList; +import meerkat.protobuf.BulletinBoardAPI.MessageFilterList; +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; +import meerkat.rest.Constants; + +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; + +import java.util.List; + +import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH; +import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH; + +/** + * Created by Arbel Deutsch Peled on 27-Dec-15. + */ +public class SingleServerReadMessagesWorker extends SingleServerWorker> { + + public SingleServerReadMessagesWorker(String serverAddress, MessageFilterList payload, int maxRetry) { + super(serverAddress, payload, maxRetry); + } + + /** + * This method carries out the actual communication with the server via HTTP Post + * Upon successful retrieval from the server the method returns the received values + * @return The list of messages returned by the server + * @throws CommunicationException if the server's response is invalid + */ + public List call() throws CommunicationException{ + + Client client = clientLocal.get(); + + WebTarget webTarget; + Response response; + + // Send request to Server + webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); + response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post( + Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF)); + + // Retrieve answer + + try { + + // If a BulletinBoardMessageList is returned: the read was successful + return response.readEntity(BulletinBoardMessageList.class).getMessageList(); + + } catch (ProcessingException | IllegalStateException e) { + + // Read failed + throw new CommunicationException("Could not contact the server"); + + } + finally { + response.close(); + } + + + } + +} diff --git a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java b/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java deleted file mode 100644 index dda76c7..0000000 --- a/bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java +++ /dev/null @@ -1,214 +0,0 @@ -import com.google.protobuf.ByteString; -import meerkat.bulletinboard.BulletinBoardClient; -import meerkat.bulletinboard.BulletinBoardClient.ClientCallback; -import meerkat.bulletinboard.ThreadedBulletinBoardClient; -import meerkat.protobuf.BulletinBoardAPI.*; -import meerkat.protobuf.Crypto; - -import meerkat.protobuf.Voting.*; -import meerkat.util.BulletinBoardMessageComparator; - -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.*; -import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.number.OrderingComparison.*; - -import java.util.*; -import java.util.concurrent.Semaphore; - -/** - * Created by Arbel Deutsch Peled on 05-Dec-15. - */ -public class BulletinBoardClientIntegrationTest { - - Semaphore jobSemaphore; - Vector thrown; - - private class PostCallback implements ClientCallback{ - - @Override - public void handleCallback(Object msg) { - System.err.println("Post operation completed"); - jobSemaphore.release(); - } - - @Override - public void handleFailure(Throwable t) { - thrown.add(t); - jobSemaphore.release(); - } - } - - private class RedundancyCallback implements ClientCallback{ - - private float minRedundancy; - - public RedundancyCallback(float minRedundancy) { - this.minRedundancy = minRedundancy; - } - - @Override - public void handleCallback(Float redundancy) { - System.err.println("Redundancy found is: " + redundancy); - jobSemaphore.release(); - assertThat(redundancy, greaterThanOrEqualTo(minRedundancy)); - } - - @Override - public void handleFailure(Throwable t) { - thrown.add(t); - jobSemaphore.release(); - } - } - - private class ReadCallback implements ClientCallback>{ - - private List expectedMsgList; - - public ReadCallback(List expectedMsgList) { - this.expectedMsgList = expectedMsgList; - } - - @Override - public void handleCallback(List messages) { - - System.err.println(messages); - jobSemaphore.release(); - - BulletinBoardMessageComparator msgComparator = new BulletinBoardMessageComparator(); - - assertThat(messages.size(), is(expectedMsgList.size())); - - Iterator expectedMessageIterator = expectedMsgList.iterator(); - Iterator receivedMessageIterator = messages.iterator(); - - while (expectedMessageIterator.hasNext()) { - assertThat(msgComparator.compare(expectedMessageIterator.next(), receivedMessageIterator.next()), is(0)); - } - - } - - @Override - public void handleFailure(Throwable t) { - thrown.add(t); - jobSemaphore.release(); - } - } - - private BulletinBoardClient bulletinBoardClient; - - private PostCallback postCallback; - private RedundancyCallback redundancyCallback; - private ReadCallback readCallback; - - private static String PROP_GETTY_URL = "gretty.httpBaseURI"; - private static String DEFAULT_BASE_URL = "http://localhost:8081"; - private static String BASE_URL = System.getProperty(PROP_GETTY_URL, DEFAULT_BASE_URL); - - @Before - public void init(){ - - bulletinBoardClient = new ThreadedBulletinBoardClient(); - - List testDB = new LinkedList(); - testDB.add(BASE_URL); - - bulletinBoardClient.init(BulletinBoardClientParams.newBuilder() - .addBulletinBoardAddress("http://localhost:8081") - .setMinRedundancy((float) 1.0) - .build()); - - postCallback = new PostCallback(); - redundancyCallback = new RedundancyCallback((float) 1.0); - - thrown = new Vector<>(); - jobSemaphore = new Semaphore(0); - - } - - @Test - public void postTest() { - - byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4}; - byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14}; - byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24}; - byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0}; - - BulletinBoardMessage msg; - - MessageFilterList filterList; - List msgList; - - MessageID messageID; - - Comparator msgComparator = new BulletinBoardMessageComparator(); - - msg = BulletinBoardMessage.newBuilder() - .setMsg(UnsignedBulletinBoardMessage.newBuilder() - .addTag("Signature") - .addTag("Trustee") - .setData(ByteString.copyFrom(b1)) - .build()) - .addSig(Crypto.Signature.newBuilder() - .setType(Crypto.SignatureType.DSA) - .setData(ByteString.copyFrom(b2)) - .setSignerId(ByteString.copyFrom(b3)) - .build()) - .addSig(Crypto.Signature.newBuilder() - .setType(Crypto.SignatureType.ECDSA) - .setData(ByteString.copyFrom(b3)) - .setSignerId(ByteString.copyFrom(b2)) - .build()) - .build(); - - messageID = bulletinBoardClient.postMessage(msg,postCallback); - - try { - jobSemaphore.acquire(); - } catch (InterruptedException e) { - System.err.println(e.getCause() + " " + e.getMessage()); - } - - bulletinBoardClient.getRedundancy(messageID,redundancyCallback); - - filterList = MessageFilterList.newBuilder() - .addFilter( - MessageFilter.newBuilder() - .setType(FilterType.TAG) - .setTag("Signature") - .build() - ) - .addFilter( - MessageFilter.newBuilder() - .setType(FilterType.TAG) - .setTag("Trustee") - .build() - ) - .build(); - - msgList = new LinkedList(); - msgList.add(msg); - - readCallback = new ReadCallback(msgList); - - bulletinBoardClient.readMessages(filterList, readCallback); - try { - jobSemaphore.acquire(2); - } catch (InterruptedException e) { - System.err.println(e.getCause() + " " + e.getMessage()); - } - - bulletinBoardClient.close(); - - for (Throwable t : thrown) { - System.err.println(t.getMessage()); - } - if (thrown.size() > 0) { - assert false; - } - - } - -} diff --git a/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java b/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java new file mode 100644 index 0000000..dfccebc --- /dev/null +++ b/bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java @@ -0,0 +1,538 @@ +import com.google.common.util.concurrent.FutureCallback; +import com.google.protobuf.ByteString; +import meerkat.bulletinboard.AsyncBulletinBoardClient; +import meerkat.bulletinboard.CompleteBatch; +import meerkat.bulletinboard.GenericBatchDigitalSignature; +import meerkat.bulletinboard.ThreadedBulletinBoardClient; +import meerkat.comm.CommunicationException; +import meerkat.crypto.concrete.ECDSASignature; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Crypto; + +import meerkat.protobuf.Voting.*; +import meerkat.util.BulletinBoardMessageComparator; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.number.OrderingComparison.*; + +import java.io.IOException; +import java.io.InputStream; +import java.security.*; +import java.security.cert.CertificateException; +import java.util.*; +import java.util.concurrent.Semaphore; + +/** + * Created by Arbel Deutsch Peled on 05-Dec-15. + */ +public class ThreadedBulletinBoardClientIntegrationTest { + + // Signature resources + + private GenericBatchDigitalSignature signers[]; + private ByteString[] signerIDs; + + private static String KEYFILE_EXAMPLE = "/certs/enduser-certs/user1-key-with-password-secret.p12"; + private static String KEYFILE_EXAMPLE3 = "/certs/enduser-certs/user3-key-with-password-shh.p12"; + + private static String KEYFILE_PASSWORD1 = "secret"; + private static String KEYFILE_PASSWORD3 = "shh"; + + private static String CERT1_PEM_EXAMPLE = "/certs/enduser-certs/user1.crt"; + private static String CERT3_PEM_EXAMPLE = "/certs/enduser-certs/user3.crt"; + + // Server data + + private static String PROP_GETTY_URL = "gretty.httpBaseURI"; + private static String DEFAULT_BASE_URL = "http://localhost:8081"; + private static String BASE_URL = System.getProperty(PROP_GETTY_URL, DEFAULT_BASE_URL); + + // Client and callbacks + + private AsyncBulletinBoardClient bulletinBoardClient; + + private PostCallback postCallback; + private PostCallback failPostCallback = new PostCallback(true,false); + + private RedundancyCallback redundancyCallback; + private ReadCallback readCallback; + private ReadBatchCallback readBatchCallback; + + // Sync and misc + + private Semaphore jobSemaphore; + private Vector thrown; + private Random random; + + // Constructor + + public ThreadedBulletinBoardClientIntegrationTest(){ + + signers = new GenericBatchDigitalSignature[2]; + signerIDs = new ByteString[signers.length]; + signers[0] = new GenericBatchDigitalSignature(new ECDSASignature()); + signers[1] = new GenericBatchDigitalSignature(new ECDSASignature()); + + InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE); + char[] password = KEYFILE_PASSWORD1.toCharArray(); + + KeyStore.Builder keyStoreBuilder; + try { + keyStoreBuilder = signers[0].getPKCS12KeyStoreBuilder(keyStream, password); + + signers[0].loadSigningCertificate(keyStoreBuilder); + + signers[0].loadVerificationCertificates(getClass().getResourceAsStream(CERT1_PEM_EXAMPLE)); + + keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE3); + password = KEYFILE_PASSWORD3.toCharArray(); + + keyStoreBuilder = signers[1].getPKCS12KeyStoreBuilder(keyStream, password); + signers[1].loadSigningCertificate(keyStoreBuilder); + + signers[1].loadVerificationCertificates(getClass().getResourceAsStream(CERT3_PEM_EXAMPLE)); + + for (int i = 0 ; i < signers.length ; i++) { + signerIDs[i] = signers[i].getSignerID(); + } + + } catch (IOException e) { + System.err.println("Failed reading from signature file " + e.getMessage()); + fail("Failed reading from signature file " + e.getMessage()); + } catch (CertificateException e) { + System.err.println("Failed reading certificate " + e.getMessage()); + fail("Failed reading certificate " + e.getMessage()); + } catch (KeyStoreException e) { + System.err.println("Failed reading keystore " + e.getMessage()); + fail("Failed reading keystore " + e.getMessage()); + } catch (NoSuchAlgorithmException e) { + System.err.println("Couldn't find signing algorithm " + e.getMessage()); + fail("Couldn't find signing algorithm " + e.getMessage()); + } catch (UnrecoverableKeyException e) { + System.err.println("Couldn't find signing key " + e.getMessage()); + fail("Couldn't find signing key " + e.getMessage()); + } + + } + + // Callback definitions + + protected void genericHandleFailure(Throwable t){ + System.err.println(t.getCause() + " " + t.getMessage()); + thrown.add(t); + jobSemaphore.release(); + } + + private class PostCallback implements FutureCallback{ + + private boolean isAssert; + private boolean assertValue; + + public PostCallback() { + this(false); + } + + public PostCallback(boolean isAssert) { + this(isAssert,true); + } + + public PostCallback(boolean isAssert, boolean assertValue) { + this.isAssert = isAssert; + this.assertValue = assertValue; + } + + @Override + public void onSuccess(Boolean msg) { + System.err.println("Post operation completed"); + jobSemaphore.release(); + //TODO: Change Assert mechanism to exception one + if (isAssert) { + if (assertValue) { + assertThat("Post operation failed", msg, is(Boolean.TRUE)); + } else { + assertThat("Post operation succeeded unexpectedly", msg, is(Boolean.FALSE)); + } + } + } + + @Override + public void onFailure(Throwable t) { + genericHandleFailure(t); + } + } + + private class RedundancyCallback implements FutureCallback{ + + private float minRedundancy; + + public RedundancyCallback(float minRedundancy) { + this.minRedundancy = minRedundancy; + } + + @Override + public void onSuccess(Float redundancy) { + System.err.println("Redundancy found is: " + redundancy); + jobSemaphore.release(); + assertThat(redundancy, greaterThanOrEqualTo(minRedundancy)); + } + + @Override + public void onFailure(Throwable t) { + genericHandleFailure(t); + } + } + + private class ReadCallback implements FutureCallback>{ + + private List expectedMsgList; + + public ReadCallback(List expectedMsgList) { + this.expectedMsgList = expectedMsgList; + } + + @Override + public void onSuccess(List messages) { + + System.err.println(messages); + jobSemaphore.release(); + + BulletinBoardMessageComparator msgComparator = new BulletinBoardMessageComparator(); + + assertThat(messages.size(), is(expectedMsgList.size())); + + Iterator expectedMessageIterator = expectedMsgList.iterator(); + Iterator receivedMessageIterator = messages.iterator(); + + while (expectedMessageIterator.hasNext()) { + assertThat(msgComparator.compare(expectedMessageIterator.next(), receivedMessageIterator.next()), is(0)); + } + + } + + @Override + public void onFailure(Throwable t) { + genericHandleFailure(t); + } + } + + private class ReadBatchCallback implements FutureCallback { + + private CompleteBatch expectedBatch; + + public ReadBatchCallback(CompleteBatch expectedBatch) { + this.expectedBatch = expectedBatch; + } + + @Override + public void onSuccess(CompleteBatch batch) { + + System.err.println(batch); + jobSemaphore.release(); + + assertThat("Batch returned is incorrect", batch, is(equalTo(expectedBatch))); + + } + + @Override + public void onFailure(Throwable t) { + genericHandleFailure(t); + } + } + + // Randomness generators + + private byte randomByte(){ + return (byte) random.nextInt(); + } + + private byte[] randomByteArray(int length) { + + byte[] randomBytes = new byte[length]; + + for (int i = 0; i < length ; i++){ + randomBytes[i] = randomByte(); + } + + return randomBytes; + + } + + private CompleteBatch createRandomBatch(int signer, int batchId, int length) throws SignatureException { + + CompleteBatch completeBatch = new CompleteBatch(); + + // Create data + + completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder() + .setSignerId(signerIDs[signer]) + .setBatchId(batchId) + .addTag("Test") + .build()); + + for (int i = 0 ; i < length ; i++){ + + BatchData batchData = BatchData.newBuilder() + .setData(ByteString.copyFrom(randomByteArray(i))) + .build(); + + completeBatch.appendBatchData(batchData); + + } + + signers[signer].updateContent(completeBatch); + + completeBatch.setSignature(signers[signer].sign()); + + return completeBatch; + + } + + // Test methods + + /** + * Takes care of initializing the client and the test resources + */ + @Before + public void init(){ + + bulletinBoardClient = new ThreadedBulletinBoardClient(); + + random = new Random(0); // We use insecure randomness in tests for repeatability + + List testDB = new LinkedList<>(); + testDB.add(BASE_URL); + + bulletinBoardClient.init(BulletinBoardClientParams.newBuilder() + .addAllBulletinBoardAddress(testDB) + .setMinRedundancy((float) 1.0) + .build()); + + postCallback = new PostCallback(); + redundancyCallback = new RedundancyCallback((float) 1.0); + + thrown = new Vector<>(); + jobSemaphore = new Semaphore(0); + + } + + /** + * Closes the client and makes sure the test fails when an exception occurred in a separate thread + */ + + @After + public void close() { + + bulletinBoardClient.close(); + + if (thrown.size() > 0) { + assert false; + } + + } + + /** + * Tests the standard post, redundancy and read methods + */ + @Test + public void postTest() { + + byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4}; + byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14}; + byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24}; + + BulletinBoardMessage msg; + + MessageFilterList filterList; + List msgList; + + MessageID messageID; + + msg = BulletinBoardMessage.newBuilder() + .setMsg(UnsignedBulletinBoardMessage.newBuilder() + .addTag("Signature") + .addTag("Trustee") + .setData(ByteString.copyFrom(b1)) + .build()) + .addSig(Crypto.Signature.newBuilder() + .setType(Crypto.SignatureType.DSA) + .setData(ByteString.copyFrom(b2)) + .setSignerId(ByteString.copyFrom(b3)) + .build()) + .addSig(Crypto.Signature.newBuilder() + .setType(Crypto.SignatureType.ECDSA) + .setData(ByteString.copyFrom(b3)) + .setSignerId(ByteString.copyFrom(b2)) + .build()) + .build(); + + messageID = bulletinBoardClient.postMessage(msg,postCallback); + + try { + jobSemaphore.acquire(); + } catch (InterruptedException e) { + System.err.println(e.getCause() + " " + e.getMessage()); + } + + bulletinBoardClient.getRedundancy(messageID,redundancyCallback); + + filterList = MessageFilterList.newBuilder() + .addFilter( + MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag("Signature") + .build() + ) + .addFilter( + MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag("Trustee") + .build() + ) + .build(); + + msgList = new LinkedList<>(); + msgList.add(msg); + + readCallback = new ReadCallback(msgList); + + bulletinBoardClient.readMessages(filterList, readCallback); + try { + jobSemaphore.acquire(2); + } catch (InterruptedException e) { + System.err.println(e.getCause() + " " + e.getMessage()); + } + + } + + /** + * Tests posting a batch by parts + * Also tests not being able to post to a closed batch + * @throws CommunicationException, SignatureException, InterruptedException + */ + @Test + public void testBatchPost() throws CommunicationException, SignatureException, InterruptedException { + + final int SIGNER = 1; + final int BATCH_ID = 100; + final int BATCH_LENGTH = 100; + + CompleteBatch completeBatch = createRandomBatch(SIGNER, BATCH_ID, BATCH_LENGTH); + + // Begin batch + + bulletinBoardClient.beginBatch(completeBatch.getBeginBatchMessage(), postCallback); + + jobSemaphore.acquire(); + + // Post data + + bulletinBoardClient.postBatchData(signerIDs[SIGNER], BATCH_ID, completeBatch.getBatchDataList(), postCallback); + + jobSemaphore.acquire(); + + // Close batch + + CloseBatchMessage closeBatchMessage = CloseBatchMessage.newBuilder() + .setBatchId(BATCH_ID) + .setBatchLength(BATCH_LENGTH) + .setSig(completeBatch.getSignature()) + .build(); + + bulletinBoardClient.closeBatch(closeBatchMessage, postCallback); + + jobSemaphore.acquire(); + + // Attempt to open batch again + + bulletinBoardClient.beginBatch(completeBatch.getBeginBatchMessage(), failPostCallback); + + // Attempt to add batch data + + bulletinBoardClient.postBatchData(signerIDs[SIGNER], BATCH_ID, completeBatch.getBatchDataList(), failPostCallback); + + jobSemaphore.acquire(2); + + // Read batch data + + BatchSpecificationMessage batchSpecificationMessage = + BatchSpecificationMessage.newBuilder() + .setSignerId(signerIDs[SIGNER]) + .setBatchId(BATCH_ID) + .setStartPosition(0) + .build(); + + readBatchCallback = new ReadBatchCallback(completeBatch); + + bulletinBoardClient.readBatch(batchSpecificationMessage, readBatchCallback); + + jobSemaphore.acquire(); + + } + + /** + * Posts a complete batch message + * Checks reading od the message + * @throws CommunicationException, SignatureException, InterruptedException + */ + @Test + public void testCompleteBatchPost() throws CommunicationException, SignatureException, InterruptedException { + + final int SIGNER = 0; + final int BATCH_ID = 101; + final int BATCH_LENGTH = 50; + + // Post batch + + CompleteBatch completeBatch = createRandomBatch(SIGNER, BATCH_ID, BATCH_LENGTH); + + bulletinBoardClient.postBatch(completeBatch,postCallback); + + jobSemaphore.acquire(); + + // Read batch + + BatchSpecificationMessage batchSpecificationMessage = + BatchSpecificationMessage.newBuilder() + .setSignerId(signerIDs[SIGNER]) + .setBatchId(BATCH_ID) + .setStartPosition(0) + .build(); + + readBatchCallback = new ReadBatchCallback(completeBatch); + + bulletinBoardClient.readBatch(batchSpecificationMessage, readBatchCallback); + + jobSemaphore.acquire(); + + } + + /** + * Tests that an unopened batch cannot be closed + * @throws CommunicationException, InterruptedException + */ + @Test + public void testInvalidBatchClose() throws CommunicationException, InterruptedException { + + final int NON_EXISTENT_BATCH_ID = 999; + + CloseBatchMessage closeBatchMessage = + CloseBatchMessage.newBuilder() + .setBatchId(NON_EXISTENT_BATCH_ID) + .setBatchLength(1) + .setSig(Crypto.Signature.getDefaultInstance()) + .build(); + + // Try to close the (unopened) batch; + + bulletinBoardClient.closeBatch(closeBatchMessage, failPostCallback); + + jobSemaphore.acquire(); + + } + +} diff --git a/bulletin-board-server/build.gradle b/bulletin-board-server/build.gradle index 0bc5452..8d824e7 100644 --- a/bulletin-board-server/build.gradle +++ b/bulletin-board-server/build.gradle @@ -48,7 +48,7 @@ dependencies { // JDBC connections compile 'org.springframework:spring-jdbc:4.2.+' - compile 'org.xerial:sqlite-jdbc:3.7.+' + compile 'org.xerial:sqlite-jdbc:3.8.+' compile 'mysql:mysql-connector-java:5.1.+' compile 'com.h2database:h2:1.0.+' @@ -79,9 +79,30 @@ test { exclude '**/*IntegrationTest*' } +task myTest(type: Test) { + include '**/*MySQL*Test*' + outputs.upToDateWhen { false } +} + +task h2Test(type: Test) { + include '**/*H2*Test*' + outputs.upToDateWhen { false } +} + +task liteTest(type: Test) { + include '**/*SQLite*Test*' + outputs.upToDateWhen { false } +} + task dbTest(type: Test) { include '**/*H2*Test*' - include '**/*MySql*Test' + include '**/*MySQL*Test*' + include '**/*SQLite*Test*' + outputs.upToDateWhen { false } +} + +task manualIntegration(type: Test) { + include '**/*IntegrationTest*' } task integrationTest(type: Test) { diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java index 52bf42b..b4e2949 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java @@ -3,26 +3,36 @@ package meerkat.bulletinboard.sqlserver; import java.sql.*; import java.util.*; -import com.google.protobuf.ProtocolStringList; +import com.google.protobuf.*; + +import meerkat.bulletinboard.*; +import meerkat.bulletinboard.sqlserver.mappers.*; +import static meerkat.bulletinboard.BulletinBoardConstants.*; -import meerkat.bulletinboard.BulletinBoardServer; -import meerkat.bulletinboard.sqlserver.mappers.EntryNumMapper; -import meerkat.bulletinboard.sqlserver.mappers.MessageMapper; -import meerkat.bulletinboard.sqlserver.mappers.SignatureMapper; import meerkat.comm.CommunicationException; + +import meerkat.comm.MessageOutputStream; +import meerkat.crypto.concrete.ECDSASignature; +import meerkat.crypto.concrete.SHA256Digest; + import meerkat.protobuf.BulletinBoardAPI.*; import meerkat.protobuf.Crypto.Signature; import meerkat.protobuf.Crypto.SignatureVerificationKey; -import meerkat.crypto.Digest; -import meerkat.crypto.concrete.SHA256Digest; + + +import static meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider.*; import javax.sql.DataSource; +import meerkat.util.BulletinBoardUtils; +import meerkat.util.TimestampComparator; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.jdbc.support.GeneratedKeyHolder; import org.springframework.jdbc.support.KeyHolder; + + /** * This is a generic SQL implementation of the BulletinBoardServer API. */ @@ -40,24 +50,120 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ */ public static enum QueryType { - FIND_MSG_ID(new String[] {"MsgId"}), - INSERT_MSG(new String[] {"MsgId","Msg"}), - INSERT_NEW_TAG(new String[] {"Tag"}), - CONNECT_TAG(new String[] {"EntryNum","Tag"}), - ADD_SIGNATURE(new String[] {"EntryNum","SignerId","Signature"}), - GET_SIGNATURES(new String[] {"EntryNum"}), - GET_MESSAGES(new String[] {}); + FIND_MSG_ID( + new String[] {"MsgId"}, + new int[] {Types.BLOB} + ), + + FIND_TAG_ID( + new String[] {"Tag"}, + new int[] {Types.VARCHAR} + ), + + INSERT_MSG( + new String[] {"MsgId","TimeStamp","Msg"}, + new int[] {Types.BLOB, Types.TIMESTAMP, Types.BLOB} + ), + + INSERT_NEW_TAG( + new String[] {"Tag"}, + new int[] {Types.VARCHAR} + ), + + CONNECT_TAG( + new String[] {"EntryNum","Tag"}, + new int[] {Types.INTEGER, Types.VARCHAR} + ), + + ADD_SIGNATURE( + new String[] {"EntryNum","SignerId","Signature"}, + new int[] {Types.INTEGER, Types.BLOB, Types.BLOB} + ), + + GET_SIGNATURES( + new String[] {"EntryNum"}, + new int[] {Types.INTEGER} + ), + + GET_MESSAGES( + new String[] {}, + new int[] {} + ), + + COUNT_MESSAGES( + new String[] {}, + new int[] {} + ), + + GET_MESSAGE_STUBS( + new String[] {}, + new int[] {} + ), + + GET_LAST_MESSAGE_ENTRY( + new String[] {}, + new int[] {} + ), + + GET_BATCH_MESSAGE_ENTRY( + new String[] {"SignerId", "BatchId"}, + new int[] {Types.BLOB, Types.INTEGER} + ), + + CHECK_BATCH_LENGTH( + new String[] {"SignerId", "BatchId"}, + new int[] {Types.BLOB, Types.INTEGER} + ), + + GET_BATCH_MESSAGE_DATA( + new String[] {"SignerId", "BatchId", "StartPosition"}, + new int[] {Types.BLOB, Types.INTEGER, Types.INTEGER} + ), + + INSERT_BATCH_DATA( + new String[] {"SignerId", "BatchId", "SerialNum", "Data"}, + new int[] {Types.BLOB, Types.INTEGER, Types.INTEGER, Types.BLOB} + ), + + CONNECT_BATCH_TAG( + new String[] {"SignerId", "BatchId", "Tag"}, + new int[] {Types.BLOB, Types.INTEGER, Types.VARCHAR} + ), + + GET_BATCH_TAGS( + new String[] {"SignerId", "BatchId"}, + new int[] {Types.BLOB, Types.INTEGER} + ), + + REMOVE_BATCH_TAGS( + new String[] {"SignerId", "BatchId"}, + new int[] {Types.BLOB, Types.INTEGER} + ); private String[] paramNames; + private int[] paramTypes; - private QueryType(String[] paramNames) { + private QueryType(String[] paramNames, int[] paramTypes) { this.paramNames = paramNames; + this.paramTypes = paramTypes; } public String[] getParamNames() { return paramNames; } + public String getParamName(int num) { + return paramNames[num]; + } + + public int[] getParamTypes() { + return paramTypes; + } + + public int getParamType(int num) { + return paramTypes[num]; + } + } /** @@ -69,7 +175,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ MSG_ID("MsgId", Types.BLOB), SIGNER_ID("SignerId", Types.BLOB), TAG("Tag", Types.VARCHAR), - LIMIT("Limit", Types.INTEGER); + LIMIT("Limit", Types.INTEGER), + TIMESTAMP("TimeStamp", Types.TIMESTAMP); private FilterTypeParam(String paramName, int paramType) { this.paramName = paramName; @@ -85,8 +192,9 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ case MSG_ID: return MSG_ID; - case EXACT_ENTRY: // Go through - case MAX_ENTRY: + case EXACT_ENTRY: // Go through + case MAX_ENTRY: // Go through + case MIN_ENTRY: return ENTRY_NUM; case SIGNER_ID: @@ -98,6 +206,10 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ case MAX_MESSAGES: return LIMIT; + case BEFORE_TIME: // Go through + case AFTER_TIME: + return TIMESTAMP; + default: return null; } @@ -152,16 +264,22 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } + /** + * This method returns the value of the parameter specified in a message filter + * @param messageFilter is the filter + * @return the object parameter for the SQL query embedded in the filter (this depends on the filter type) + */ private Object getParam(MessageFilter messageFilter) { switch (messageFilter.getType()) { - case MSG_ID: // Go through + case MSG_ID: // Go through case SIGNER_ID: return messageFilter.getId().toByteArray(); - case EXACT_ENTRY: // Go through - case MAX_ENTRY: + case EXACT_ENTRY: // Go through + case MAX_ENTRY: // Go through + case MIN_ENTRY: return messageFilter.getEntry(); case TAG: @@ -170,7 +288,11 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ case MAX_MESSAGES: return messageFilter.getMaxMessages(); - default: + case BEFORE_TIME: // Go through + case AFTER_TIME: + return BulletinBoardUtils.toSQLTimestamp(messageFilter.getTimestamp()); + + default: // Unsupported filter type return null; } @@ -193,7 +315,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ protected NamedParameterJdbcTemplate jdbcTemplate; - protected Digest digest; + protected BatchDigest digest; + protected BatchDigitalSignature signer; protected List trusteeSignatureVerificationArray; protected int minTrusteeSignatures; @@ -216,8 +339,6 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ */ private void createSchema() throws SQLException { - final int TIMEOUT = 20; - for (String command : sqlQueryProvider.getSchemaCreationCommands()) { jdbcTemplate.update(command,(Map) null); } @@ -231,7 +352,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ public void init(String meerkatDB) throws CommunicationException { // TODO write signature reading part. - digest = new SHA256Digest(); + digest = new GenericBatchDigest(new SHA256Digest()); + signer = new GenericBatchDigitalSignature(new ECDSASignature()); jdbcTemplate = new NamedParameterJdbcTemplate(sqlQueryProvider.getDataSource()); @@ -264,12 +386,12 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ String sql; - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.INSERT_NEW_TAG); + sql = sqlQueryProvider.getSQLString(QueryType.INSERT_NEW_TAG); Map namedParameters[] = new HashMap[tags.length]; for (int i = 0 ; i < tags.length ; i++){ namedParameters[i] = new HashMap(); - namedParameters[i].put("Tag", tags[i]); + namedParameters[i].put(QueryType.INSERT_NEW_TAG.getParamName(0), tags[i]); } jdbcTemplate.batchUpdate(sql, namedParameters); @@ -287,10 +409,17 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ .build(); } - @Override - public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException { - - if (!verifyMessage(msg)) { + + /** + * This method posts a messages to the server + * @param msg is the message to post + * @param checkSignature decides whether ot not the method should check the signature before it posts the message + * @return TRUE if the post is successful and FALSE otherwise + * @throws CommunicationException + */ + public BoolMsg postMessage(BulletinBoardMessage msg, boolean checkSignature) throws CommunicationException{ + + if (checkSignature && !verifyMessage(msg)) { return boolToBoolMsg(false); } @@ -299,28 +428,27 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ byte[] msgID; long entryNum; - + ProtocolStringList tagList; String[] tags; - + List signatureList; Signature[] signatures; - + // Calculate message ID (depending only on the the unsigned message) - + digest.reset(); digest.update(msg.getMsg()); - + msgID = digest.digest(); - + // Add message to table if needed and store entry number of message. - - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.FIND_MSG_ID); + sql = sqlQueryProvider.getSQLString(QueryType.FIND_MSG_ID); Map namedParameters = new HashMap(); - namedParameters.put("MsgId",msgID); + namedParameters.put(QueryType.FIND_MSG_ID.getParamName(0),msgID); - List entryNums = jdbcTemplate.query(sql, new MapSqlParameterSource(namedParameters), new EntryNumMapper()); + List entryNums = jdbcTemplate.query(sql, new MapSqlParameterSource(namedParameters), new LongMapper()); if (entryNums.size() > 0){ @@ -328,8 +456,10 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } else{ - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.INSERT_MSG); - namedParameters.put("Msg", msg.getMsg().toByteArray()); + sql = sqlQueryProvider.getSQLString(QueryType.INSERT_MSG); + + namedParameters.put(QueryType.INSERT_MSG.getParamName(1), BulletinBoardUtils.toSQLTimestamp(msg.getMsg().getTimestamp())); + namedParameters.put(QueryType.INSERT_MSG.getParamName(2), msg.getMsg().toByteArray()); KeyHolder keyHolder = new GeneratedKeyHolder(); jdbcTemplate.update(sql,new MapSqlParameterSource(namedParameters),keyHolder); @@ -337,90 +467,95 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ entryNum = keyHolder.getKey().longValue(); } - + // Retrieve tags and store new ones in tag table. - + try { - - tagList = msg.getMsg().getTagList(); - tags = new String[tagList.size()]; - tags = tagList.toArray(tags); - - insertNewTags(tags); - + + tagList = msg.getMsg().getTagList(); + tags = new String[tagList.size()]; + tags = tagList.toArray(tags); + + insertNewTags(tags); + } catch (SQLException e) { throw new CommunicationException(e.getMessage()); } - + // Connect message to tags. - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.CONNECT_TAG); + sql = sqlQueryProvider.getSQLString(QueryType.CONNECT_TAG); namedParameterArray = new HashMap[tags.length]; for (int i = 0 ; i < tags.length ; i++) { namedParameterArray[i] = new HashMap(); - namedParameterArray[i].put("EntryNum", entryNum); - namedParameterArray[i].put("Tag", tags[i]); + namedParameterArray[i].put(QueryType.CONNECT_TAG.getParamName(0), entryNum); + namedParameterArray[i].put(QueryType.CONNECT_TAG.getParamName(1), tags[i]); } jdbcTemplate.batchUpdate(sql, namedParameterArray); - + // Retrieve signatures. - - signatureList = msg.getSigList(); - signatures = new Signature[signatureList.size()]; - signatures = signatureList.toArray(signatures); - + + signatureList = msg.getSigList(); + signatures = new Signature[signatureList.size()]; + signatures = signatureList.toArray(signatures); + // Connect message to signatures. - sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.ADD_SIGNATURE); + sql = sqlQueryProvider.getSQLString(QueryType.ADD_SIGNATURE); namedParameterArray = new HashMap[signatures.length]; for (int i = 0 ; i < signatures.length ; i++) { namedParameterArray[i] = new HashMap(); - namedParameterArray[i].put("EntryNum", entryNum); - namedParameterArray[i].put("SignerId", signatures[i].getSignerId().toByteArray()); - namedParameterArray[i].put("Signature", signatures[i].toByteArray()); + namedParameterArray[i].put(QueryType.ADD_SIGNATURE.getParamName(0), entryNum); + namedParameterArray[i].put(QueryType.ADD_SIGNATURE.getParamName(1), signatures[i].getSignerId().toByteArray()); + namedParameterArray[i].put(QueryType.ADD_SIGNATURE.getParamName(2), signatures[i].toByteArray()); } jdbcTemplate.batchUpdate(sql,namedParameterArray); return boolToBoolMsg(true); + } @Override - public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException { + public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException { + return postMessage(msg, true); // Perform a post and check the signature for authenticity + } - BulletinBoardMessageList.Builder resultListBuilder = BulletinBoardMessageList.newBuilder(); - // SQL length is roughly 50 characters per filter + 50 for the query itself - StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1)); + /** + * This is a container class for and SQL string builder and a MapSqlParameterSource to be used with it + */ + class SQLAndParameters { - MapSqlParameterSource namedParameters; - int paramNum; + public StringBuilder sql; + public MapSqlParameterSource parameters; - MessageMapper messageMapper = new MessageMapper(); - SignatureMapper signatureMapper = new SignatureMapper(); + public SQLAndParameters(int numOfFilters) { + sql = new StringBuilder(50 * numOfFilters); + parameters = new MapSqlParameterSource(); + } + + } + + SQLAndParameters getSQLFromFilters(MessageFilterList filterList) { + + SQLAndParameters result = new SQLAndParameters(filterList.getFilterCount()); List filters = new ArrayList(filterList.getFilterList()); - boolean isFirstFilter = true; - Collections.sort(filters, new FilterTypeComparator()); - // Check if Tag/Signature tables are required for filtering purposes - - sqlBuilder.append(sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.GET_MESSAGES)); - // Add conditions - - namedParameters = new MapSqlParameterSource(); + boolean isFirstFilter = true; if (!filters.isEmpty()) { - sqlBuilder.append(" WHERE "); + result.sql.append(" WHERE "); - for (paramNum = 0 ; paramNum < filters.size() ; paramNum++) { + for (int paramNum = 0 ; paramNum < filters.size() ; paramNum++) { MessageFilter filter = filters.get(paramNum); @@ -428,15 +563,15 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ if (isFirstFilter) { isFirstFilter = false; } else { - sqlBuilder.append(" AND "); + result.sql.append(" AND "); } } - sqlBuilder.append(sqlQueryProvider.getCondition(filter.getType(), paramNum)); + result.sql.append(sqlQueryProvider.getCondition(filter.getType(), paramNum)); - SQLQueryProvider.FilterTypeParam filterTypeParam = SQLQueryProvider.FilterTypeParam.getFilterTypeParamName(filter.getType()); + FilterTypeParam filterTypeParam = FilterTypeParam.getFilterTypeParamName(filter.getType()); - namedParameters.addValue( + result.parameters.addValue( filterTypeParam.getParamName() + Integer.toString(paramNum), getParam(filter), filterTypeParam.getParamType(), @@ -446,35 +581,378 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{ } + return result; + + } + + + /** + * Used to retrieve just basic information about messages to allow calculation of checksum + * @param filterList is a filter list that defines which messages the client is interested in + * @return a list of Bulletin Board Messages that contain just the entry number, timestamp and message ID for each message + * The message ID is returned inside the message data field + */ + protected List readMessageStubs(MessageFilterList filterList) { + + StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1)); + + sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.GET_MESSAGE_STUBS)); + + // Get Conditions + + SQLAndParameters sqlAndParameters = getSQLFromFilters(filterList); + + sqlBuilder.append(sqlAndParameters.sql); + // Run query - List msgBuilders = jdbcTemplate.query(sqlBuilder.toString(), namedParameters, messageMapper); + return jdbcTemplate.query(sqlBuilder.toString(), sqlAndParameters.parameters, new MessageStubMapper()); - // Compile list of messages + } - for (BulletinBoardMessage.Builder msgBuilder : msgBuilders) { - // Retrieve signatures + @Override + public void readMessages(MessageFilterList filterList, MessageOutputStream out) throws CommunicationException { - namedParameters = new MapSqlParameterSource(); - namedParameters.addValue("EntryNum", msgBuilder.getEntryNum()); + BulletinBoardMessageList.Builder resultListBuilder = BulletinBoardMessageList.newBuilder(); - List signatures = jdbcTemplate.query( - sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.GET_SIGNATURES), - namedParameters, - signatureMapper); + // SQL length is roughly 50 characters per filter + 50 for the query itself + StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1)); - // Append signatures - msgBuilder.addAllSig(signatures); + // Check if Tag/Signature tables are required for filtering purposes - // Finalize message and add to message list. + sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.GET_MESSAGES)); - resultListBuilder.addMessage(msgBuilder.build()); + // Get conditions + + SQLAndParameters sqlAndParameters = getSQLFromFilters(filterList); + sqlBuilder.append(sqlAndParameters.sql); + + // Run query and stream the output using a MessageCallbackHandler + + jdbcTemplate.query(sqlBuilder.toString(), sqlAndParameters.parameters, new MessageCallbackHandler(jdbcTemplate, sqlQueryProvider, out)); + + } + + /** + * This method returns a string representation of the tag associated with a batch ID + * @param batchId is the given batch ID + * @return the String representation of the tag + */ + private String batchIdToTag(int batchId) { + return BATCH_ID_TAG_PREFIX + Integer.toString(batchId); + } + + /** + * This method checks if a specified batch exists and is already closed + * @param signerId is the ID of the publisher of the batch + * @param batchId is the unique (per signer) batch ID + * @return TRUE if the batch is closed and FALSE if it is still open or doesn't exist at all + */ + private boolean isBatchClosed(ByteString signerId, int batchId) throws CommunicationException { + + MessageFilterList filterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.SIGNER_ID) + .setId(signerId) + .build()) + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(BATCH_TAG) + .build()) + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(batchIdToTag(batchId)) + .build()) + .build(); + + // SQL length is roughly 50 characters per filter + 50 for the query itself + StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1)); + + // Check if Tag/Signature tables are required for filtering purposes + + sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.COUNT_MESSAGES)); + + // Get conditions + + SQLAndParameters sqlAndParameters = getSQLFromFilters(filterList); + sqlBuilder.append(sqlAndParameters.sql); + + // Run query and stream the output using a MessageCallbackHandler + + List count = jdbcTemplate.query(sqlBuilder.toString(), sqlAndParameters.parameters, new LongMapper()); + + return (count.size() > 0) && (count.get(0) > 0); + + } + + @Override + public BoolMsg beginBatch(BeginBatchMessage message) throws CommunicationException { + + // Check if batch is closed + if (isBatchClosed(message.getSignerId(), message.getBatchId())) { + return BoolMsg.newBuilder().setValue(false).build(); + } + + // Add new tags to table + ProtocolStringList tagList = message.getTagList(); + String[] tags = new String[tagList.size()]; + tags = tagList.toArray(tags); + try { + insertNewTags(tags); + } catch (SQLException e) { + throw new CommunicationException(e.getMessage()); + } + + // Connect tags + String sql = sqlQueryProvider.getSQLString(QueryType.CONNECT_BATCH_TAG); + MapSqlParameterSource namedParameters[] = new MapSqlParameterSource[tags.length]; + + for (int i=0 ; i < tags.length ; i++) { + namedParameters[i] = new MapSqlParameterSource(); + namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(0),message.getSignerId().toByteArray()); + namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(1),message.getBatchId()); + namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(2),tags[i]); + } + + jdbcTemplate.batchUpdate(sql,namedParameters); + + return BoolMsg.newBuilder().setValue(true).build(); + } + + @Override + public BoolMsg postBatchMessage(BatchMessage batchMessage) throws CommunicationException{ + + // Check if batch is closed + if (isBatchClosed(batchMessage.getSignerId(), batchMessage.getBatchId())) { + return BoolMsg.newBuilder().setValue(false).build(); + } + + // Add data + String sql = sqlQueryProvider.getSQLString(QueryType.INSERT_BATCH_DATA); + MapSqlParameterSource namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(0),batchMessage.getSignerId().toByteArray()); + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(1),batchMessage.getBatchId()); + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(2),batchMessage.getSerialNum()); + namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(3),batchMessage.getData().toByteArray()); + + jdbcTemplate.update(sql, namedParameters); + + return BoolMsg.newBuilder().setValue(true).build(); + + } + + @Override + public BoolMsg closeBatchMessage(CloseBatchMessage message) throws CommunicationException { + + ByteString signerId = message.getSig().getSignerId(); + int batchId = message.getBatchId(); + + KeyHolder keyHolder = new GeneratedKeyHolder(); + + // Check batch size + + String sql = sqlQueryProvider.getSQLString(QueryType.CHECK_BATCH_LENGTH); + MapSqlParameterSource namedParameters = new MapSqlParameterSource(); + + + namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(0),signerId.toByteArray()); + namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(1),batchId); + + List lengthResult = jdbcTemplate.query(sql, namedParameters, new LongMapper()); + + if (lengthResult.get(0) != message.getBatchLength()) { + return BoolMsg.newBuilder().setValue(false).build(); + } + + + // Get Tags and add them to CompleteBatch + + sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_TAGS); + namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.GET_BATCH_TAGS.getParamName(0),signerId); + namedParameters.addValue(QueryType.GET_BATCH_TAGS.getParamName(1),batchId); + + List tags = jdbcTemplate.query(sql, namedParameters, new StringMapper()); + + CompleteBatch completeBatch = new CompleteBatch( + BeginBatchMessage.newBuilder() + .setSignerId(signerId) + .setBatchId(batchId) + .addAllTag(tags) + .build() + ); + + // Add actual batch data to CompleteBatch + + sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA); + namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),signerId.toByteArray()); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),batchId); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),0); // Read from the beginning + + completeBatch.appendBatchData(jdbcTemplate.query(sql, namedParameters, new BatchDataMapper())); + + // Verify signature + + completeBatch.setSignature(message.getSig()); + +// try { +// TODO: Actual verification +// //signer.verify(completeBatch); +// } catch (CertificateException | InvalidKeyException | SignatureException e) { +// return BoolMsg.newBuilder().setValue(false).build(); +// } + + // Batch verified: finalize it + + // Calculate message ID + digest.reset(); + digest.update(completeBatch); + MessageID msgID = MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build(); + + // Create Bulletin Board message + BulletinBoardMessage bulletinBoardMessage = BulletinBoardMessage.newBuilder() + .addSig(message.getSig()) + .setMsg(UnsignedBulletinBoardMessage.newBuilder() + .addAllTag(tags) + .addTag(BATCH_TAG) + .addTag(batchIdToTag(batchId)) + .setData(message.getSig().getSignerId()) + .setTimestamp(message.getTimestamp()) + .build()) + .build(); + + // Post message without checking signature validity + postMessage(bulletinBoardMessage, false); + + // Remove tags from temporary table + sql = sqlQueryProvider.getSQLString(QueryType.REMOVE_BATCH_TAGS); + namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.REMOVE_BATCH_TAGS.getParamName(0), signerId.toByteArray()); + namedParameters.addValue(QueryType.REMOVE_BATCH_TAGS.getParamName(1), batchId); + + jdbcTemplate.update(sql, namedParameters); + + // Return TRUE + + return BoolMsg.newBuilder().setValue(true).build(); + } + + @Override + public void readBatch(BatchSpecificationMessage message, MessageOutputStream out) throws CommunicationException, IllegalArgumentException{ + + // Check that batch is closed + if (!isBatchClosed(message.getSignerId(), message.getBatchId())) { + throw new IllegalArgumentException("No such batch"); + } + + String sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA); + MapSqlParameterSource namedParameters = new MapSqlParameterSource(); + + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),message.getSignerId().toByteArray()); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),message.getBatchId()); + namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),message.getStartPosition()); + + jdbcTemplate.query(sql, namedParameters, new BatchDataCallbackHandler(out)); + + } + + /** + * Finds the entry number of the last entry in the database + * @return the entry number, or -1 if no entries are found + */ + protected long getLastMessageEntry() { + + String sql = sqlQueryProvider.getSQLString(QueryType.GET_LAST_MESSAGE_ENTRY); + + List resultList = jdbcTemplate.query(sql, new LongMapper()); + + if (resultList.size() <= 0){ + return -1; + } + + return resultList.get(0); + + } + + /** + * Searches for the latest time of sync of the DB relative to a given query and returns the metadata needed to complete the sync + * The checksum up to (and including) each given timestamp is calculated using bitwise XOR on 8-byte sized blocks of the message IDs + * @param syncQuery contains a succinct representation of states to compare to + * @return the current last entry num and latest time of sync if there is one; -1 as last entry and empty timestamp otherwise + * @throws CommunicationException + */ + @Override + public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException { + + if (syncQuery == null){ + return SyncQueryResponse.newBuilder() + .setLastEntryNum(-1) + .setLastTimeOfSync(com.google.protobuf.Timestamp.getDefaultInstance()) + .build(); + } + + com.google.protobuf.Timestamp lastTimeOfSync = null; + + TimestampComparator timestampComparator = new TimestampComparator(); + + long lastEntryNum = getLastMessageEntry(); + + long checksum = 0; + + Iterator queryIterator = syncQuery.getQueryList().iterator(); + + SingleSyncQuery currentQuery = queryIterator.next(); + + List messageStubs = readMessageStubs(syncQuery.getFilterList()); + + for (BulletinBoardMessage message : messageStubs){ + + // Check for end of current query + if (timestampComparator.compare(message.getMsg().getTimestamp(), currentQuery.getTimeOfSync()) > 0){ + + if (checksum == currentQuery.getChecksum()){ + lastTimeOfSync = currentQuery.getTimeOfSync(); + } else { + break; + } + + if (queryIterator.hasNext()){ + currentQuery = queryIterator.next(); + } else{ + break; + } + + } + + // Advance checksum + + ByteString messageID = message.getMsg().getData(); + + checksum &= messageID.byteAt(0) & messageID.byteAt(1) & messageID.byteAt(2) & messageID.byteAt(3); } - //Combine results and return. - return resultListBuilder.build(); + if (checksum == currentQuery.getChecksum()){ + lastTimeOfSync = currentQuery.getTimeOfSync(); + } + + if (lastTimeOfSync == null){ + return SyncQueryResponse.newBuilder() + .setLastEntryNum(-1) + .setLastTimeOfSync(com.google.protobuf.Timestamp.getDefaultInstance()) + .build(); + } else{ + return SyncQueryResponse.newBuilder() + .setLastEntryNum(lastEntryNum) + .setLastTimeOfSync(lastTimeOfSync) + .build(); + } } diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java index fa2b146..c390048 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java @@ -7,6 +7,7 @@ import javax.naming.InitialContext; import javax.naming.NamingException; import javax.sql.DataSource; +import java.text.MessageFormat; import java.util.LinkedList; import java.util.List; @@ -42,9 +43,20 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider case FIND_MSG_ID: return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId"; + case FIND_TAG_ID: + return MessageFormat.format( + "SELECT TagId FROM TagTable WHERE Tag = :{0}", + QueryType.FIND_TAG_ID.getParamName(0)); + case GET_MESSAGES: return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"; + case COUNT_MESSAGES: + return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable"; + + case GET_MESSAGE_STUBS: + return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable"; + case GET_SIGNATURES: return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum"; @@ -55,6 +67,66 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider return "INSERT INTO TagTable(Tag) SELECT DISTINCT :Tag AS NewTag FROM UtilityTable WHERE" + " NOT EXISTS (SELECT 1 FROM TagTable AS SubTable WHERE SubTable.Tag = :Tag)"; + case GET_LAST_MESSAGE_ENTRY: + return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable"; + + case GET_BATCH_MESSAGE_ENTRY: + return MessageFormat.format( + "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable" + + " INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum" + + " INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum" + + " INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId" + + " WHERE SignatureTable.SignerId = :{0}" + + " AND TagTable.Tag = :{1}", + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0), + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1)); + + case GET_BATCH_MESSAGE_DATA: + return MessageFormat.format( + "SELECT Data FROM BatchTable" + + " WHERE SignerId = :{0} AND BatchId = :{1} AND SerialNum >= :{2}" + + " ORDER BY SerialNum ASC", + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2)); + + case INSERT_BATCH_DATA: + return MessageFormat.format( + "INSERT INTO BatchTable (SignerId, BatchId, SerialNum, Data)" + + " VALUES (:{0}, :{1}, :{2}, :{3})", + QueryType.INSERT_BATCH_DATA.getParamName(0), + QueryType.INSERT_BATCH_DATA.getParamName(1), + QueryType.INSERT_BATCH_DATA.getParamName(2), + QueryType.INSERT_BATCH_DATA.getParamName(3)); + + case CHECK_BATCH_LENGTH: + return MessageFormat.format( + "SELECT COUNT(Data) AS BatchLength FROM BatchTable" + + " WHERE SignerId = :{0} AND BatchId = :{1}", + QueryType.CHECK_BATCH_LENGTH.getParamName(0), + QueryType.CHECK_BATCH_LENGTH.getParamName(1)); + + case CONNECT_BATCH_TAG: + return MessageFormat.format( + "INSERT INTO BatchTagTable (SignerId, BatchId, TagId) SELECT :{0}, :{1}, TagId FROM TagTable" + + " WHERE Tag = :{2}", + QueryType.CONNECT_BATCH_TAG.getParamName(0), + QueryType.CONNECT_BATCH_TAG.getParamName(1), + QueryType.CONNECT_BATCH_TAG.getParamName(2)); + + case GET_BATCH_TAGS: + return MessageFormat.format( + "SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.TagId" + + " WHERE SignerId = :{0} AND BatchId = :{1} ORDER BY Tag ASC", + QueryType.GET_BATCH_TAGS.getParamName(0), + QueryType.GET_BATCH_TAGS.getParamName(1)); + + case REMOVE_BATCH_TAGS: + return MessageFormat.format( + "DELETE FROM BatchTagTable WHERE SignerId = :{0} AND BatchId = :{1}", + QueryType.REMOVE_BATCH_TAGS.getParamName(0), + QueryType.REMOVE_BATCH_TAGS.getParamName(1)); + default: throw new IllegalArgumentException("Cannot serve a query of type " + queryType); } @@ -71,10 +143,12 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider return "MsgTable.EntryNum = :EntryNum" + serialString; case MAX_ENTRY: return "MsgTable.EntryNum <= :EntryNum" + serialString; + case MIN_ENTRY: + return "MsgTable.EntryNum >= :EntryNum" + serialString; case MAX_MESSAGES: return "LIMIT :Limit" + serialString; case MSG_ID: - return "MsgTable.MsgId = MsgId" + serialString; + return "MsgTable.MsgId = :MsgId" + serialString; case SIGNER_ID: return "EXISTS (SELECT 1 FROM SignatureTable" + " WHERE SignatureTable.SignerId = :SignerId" + serialString + " AND SignatureTable.EntryNum = MsgTable.EntryNum)"; @@ -82,6 +156,13 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider return "EXISTS (SELECT 1 FROM TagTable" + " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId" + " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)"; + + case BEFORE_TIME: + return "MsgTable.ExactTime <= :TimeStamp"; + + case AFTER_TIME: + return "MsgTable.ExactTime >= :TimeStamp"; + default: throw new IllegalArgumentException("Cannot serve a filter of type " + filterType); } @@ -94,6 +175,7 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider switch(filterType) { case EXACT_ENTRY: // Go through case MAX_ENTRY: // Go through + case MIN_ENTRY: // Go through case MAX_MESSAGES: return "INT"; @@ -124,7 +206,7 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider public List getSchemaCreationCommands() { List list = new LinkedList(); - list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB UNIQUE, Msg BLOB)"); + list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB UNIQUE, ExactTime TIMESTAMP, Msg BLOB)"); list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INT NOT NULL AUTO_INCREMENT PRIMARY KEY, Tag VARCHAR(50) UNIQUE)"); @@ -139,6 +221,14 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider list.add("CREATE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId)"); list.add("CREATE UNIQUE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId, EntryNum)"); + list.add("CREATE TABLE IF NOT EXISTS BatchTable (SignerId TINYBLOB, BatchId INT, SerialNum INT, Data BLOB," + + " UNIQUE(SignerId, BatchId, SerialNum))"); + + list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (SignerId TINYBLOB, BatchId INT, TagId INT," + + " FOREIGN KEY (TagId) REFERENCES TagTable(TagId))"); + + list.add("CREATE INDEX IF NOT EXISTS BatchIndex ON BatchTagTable(SignerId, BatchId)"); + // This is used to create a simple table with one entry. // It is used for implementing a workaround for the missing INSERT IGNORE syntax list.add("CREATE TABLE IF NOT EXISTS UtilityTable (Entry INT)"); @@ -152,6 +242,9 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider List list = new LinkedList(); list.add("DROP TABLE IF EXISTS UtilityTable"); + list.add("DROP INDEX IF EXISTS BatchIndex"); + list.add("DROP TABLE IF EXISTS BatchTagTable"); + list.add("DROP TABLE IF EXISTS BatchTable"); list.add("DROP INDEX IF EXISTS SignerIdIndex"); list.add("DROP TABLE IF EXISTS MsgTagTable"); list.add("DROP TABLE IF EXISTS SignatureTable"); diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java index c00c044..f99114e 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java @@ -1,10 +1,12 @@ package meerkat.bulletinboard.sqlserver; import com.mysql.jdbc.jdbc2.optional.MysqlDataSource; +import meerkat.bulletinboard.BulletinBoardConstants; import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider; import meerkat.protobuf.BulletinBoardAPI.FilterType; import javax.sql.DataSource; +import java.text.MessageFormat; import java.util.LinkedList; import java.util.List; @@ -32,21 +34,117 @@ public class MySQLQueryProvider implements SQLQueryProvider { public String getSQLString(QueryType queryType) throws IllegalArgumentException{ switch(queryType) { + case ADD_SIGNATURE: - return "INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:EntryNum, :SignerId, :Signature)"; + return MessageFormat.format( + "INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:{0}, :{1}, :{2})", + QueryType.ADD_SIGNATURE.getParamName(0), + QueryType.ADD_SIGNATURE.getParamName(1), + QueryType.ADD_SIGNATURE.getParamName(2)); + case CONNECT_TAG: - return "INSERT IGNORE INTO MsgTagTable (TagId, EntryNum)" - + " SELECT TagTable.TagId, :EntryNum AS EntryNum FROM TagTable WHERE Tag = :Tag"; + return MessageFormat.format( + "INSERT IGNORE INTO MsgTagTable (TagId, EntryNum)" + + " SELECT TagTable.TagId, :{0} AS EntryNum FROM TagTable WHERE Tag = :{1}", + QueryType.CONNECT_TAG.getParamName(0), + QueryType.CONNECT_TAG.getParamName(1)); + case FIND_MSG_ID: - return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId"; + return MessageFormat.format( + "SELECT EntryNum From MsgTable WHERE MsgId = :{0}", + QueryType.FIND_MSG_ID.getParamName(0)); + + case FIND_TAG_ID: + return MessageFormat.format( + "SELECT TagId FROM TagTable WHERE Tag = :{0}", + QueryType.FIND_TAG_ID.getParamName(0)); + case GET_MESSAGES: return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"; + + case COUNT_MESSAGES: + return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable"; + + case GET_MESSAGE_STUBS: + return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable"; + case GET_SIGNATURES: - return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum"; + return MessageFormat.format( + "SELECT Signature FROM SignatureTable WHERE EntryNum = :{0}", + QueryType.GET_SIGNATURES.getParamName(0)); + case INSERT_MSG: - return "INSERT INTO MsgTable (MsgId, Msg) VALUES(:MsgId, :Msg)"; + return MessageFormat.format( + "INSERT INTO MsgTable (MsgId, ExactTime, Msg) VALUES(:{0}, :{1}, :{2})", + QueryType.INSERT_MSG.getParamName(0), + QueryType.INSERT_MSG.getParamName(1), + QueryType.INSERT_MSG.getParamName(2)); + case INSERT_NEW_TAG: - return "INSERT IGNORE INTO TagTable(Tag) VALUES (:Tag)"; + return MessageFormat.format( + "INSERT IGNORE INTO TagTable(Tag) VALUES (:{0})", + QueryType.INSERT_NEW_TAG.getParamName(0)); + + case GET_LAST_MESSAGE_ENTRY: + return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable"; + + case GET_BATCH_MESSAGE_ENTRY: + return MessageFormat.format( + "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable" + + " INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum" + + " INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum" + + " INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId" + + " WHERE SignatureTable.SignerId = :{0}" + + " AND TagTable.Tag = :{1}", + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0), + QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1)); + + case GET_BATCH_MESSAGE_DATA: + return MessageFormat.format( + "SELECT Data FROM BatchTable" + + " WHERE SignerId = :{0} AND BatchId = :{1} AND SerialNum >= :{2}" + + " ORDER BY SerialNum ASC", + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1), + QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2)); + + case INSERT_BATCH_DATA: + return MessageFormat.format( + "INSERT INTO BatchTable (SignerId, BatchId, SerialNum, Data)" + + " VALUES (:{0}, :{1}, :{2}, :{3})", + QueryType.INSERT_BATCH_DATA.getParamName(0), + QueryType.INSERT_BATCH_DATA.getParamName(1), + QueryType.INSERT_BATCH_DATA.getParamName(2), + QueryType.INSERT_BATCH_DATA.getParamName(3)); + + case CHECK_BATCH_LENGTH: + return MessageFormat.format( + "SELECT COUNT(Data) AS BatchLength FROM BatchTable" + + " WHERE SignerId = :{0} AND BatchId = :{1}", + QueryType.CHECK_BATCH_LENGTH.getParamName(0), + QueryType.CHECK_BATCH_LENGTH.getParamName(1)); + + case CONNECT_BATCH_TAG: + return MessageFormat.format( + "INSERT INTO BatchTagTable (SignerId, BatchId, TagId) SELECT :{0}, :{1}, TagId FROM TagTable" + + " WHERE Tag = :{2}", + QueryType.CONNECT_BATCH_TAG.getParamName(0), + QueryType.CONNECT_BATCH_TAG.getParamName(1), + QueryType.CONNECT_BATCH_TAG.getParamName(2)); + + case GET_BATCH_TAGS: + return MessageFormat.format( + "SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.TagId" + + " WHERE SignerId = :{0} AND BatchId = :{1} ORDER BY Tag ASC", + QueryType.GET_BATCH_TAGS.getParamName(0), + QueryType.GET_BATCH_TAGS.getParamName(1)); + + case REMOVE_BATCH_TAGS: + return MessageFormat.format( + "DELETE FROM BatchTagTable WHERE SignerId = :{0} AND BatchId = :{1}", + QueryType.REMOVE_BATCH_TAGS.getParamName(0), + QueryType.REMOVE_BATCH_TAGS.getParamName(1)); + default: throw new IllegalArgumentException("Cannot serve a query of type " + queryType); } @@ -63,6 +161,8 @@ public class MySQLQueryProvider implements SQLQueryProvider { return "MsgTable.EntryNum = :EntryNum" + serialString; case MAX_ENTRY: return "MsgTable.EntryNum <= :EntryNum" + serialString; + case MIN_ENTRY: + return "MsgTable.EntryNum >= :EntryNum" + serialString; case MAX_MESSAGES: return "LIMIT :Limit" + serialString; case MSG_ID: @@ -74,6 +174,13 @@ public class MySQLQueryProvider implements SQLQueryProvider { return "EXISTS (SELECT 1 FROM TagTable" + " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId" + " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)"; + + case BEFORE_TIME: + return "MsgTable.ExactTime <= :TimeStamp"; + + case AFTER_TIME: + return "MsgTable.ExactTime >= :TimeStamp"; + default: throw new IllegalArgumentException("Cannot serve a filter of type " + filterType); } @@ -86,6 +193,7 @@ public class MySQLQueryProvider implements SQLQueryProvider { switch(filterType) { case EXACT_ENTRY: // Go through case MAX_ENTRY: // Go through + case MIN_ENTRY: // Go through case MAX_MESSAGES: return "INT"; @@ -96,6 +204,10 @@ public class MySQLQueryProvider implements SQLQueryProvider { case TAG: return "VARCHAR"; + case AFTER_TIME: // Go through + case BEFORE_TIME: + return "TIMESTAMP"; + default: throw new IllegalArgumentException("Cannot serve a filter of type " + filterType); } @@ -111,6 +223,7 @@ public class MySQLQueryProvider implements SQLQueryProvider { dataSource.setDatabaseName(dbName); dataSource.setUser(username); dataSource.setPassword(password); + dataSource.setAllowMultiQueries(true); return dataSource; } @@ -119,7 +232,8 @@ public class MySQLQueryProvider implements SQLQueryProvider { public List getSchemaCreationCommands() { List list = new LinkedList(); - list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB, Msg BLOB, UNIQUE(MsgId(50)))"); + list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY," + + " MsgId TINYBLOB, ExactTime TIMESTAMP, Msg BLOB, UNIQUE(MsgId(50)))"); list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INT NOT NULL AUTO_INCREMENT PRIMARY KEY, Tag VARCHAR(50), UNIQUE(Tag))"); @@ -129,7 +243,14 @@ public class MySQLQueryProvider implements SQLQueryProvider { + " CONSTRAINT UNIQUE (EntryNum, TagID))"); list.add("CREATE TABLE IF NOT EXISTS SignatureTable (EntryNum INT, SignerId TINYBLOB, Signature TINYBLOB," - + " INDEX(SignerId(32)), CONSTRAINT Uni UNIQUE(SignerId(32), EntryNum), CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))"); + + " INDEX(SignerId(32)), CONSTRAINT Unique_Signature UNIQUE(SignerId(32), EntryNum)," + + " CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))"); + + list.add("CREATE TABLE IF NOT EXISTS BatchTable (SignerId TINYBLOB, BatchId INT, SerialNum INT, Data BLOB," + + " CONSTRAINT Unique_Batch UNIQUE(SignerId(32), BatchId, SerialNum))"); + + list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (SignerId TINYBLOB, BatchId INT, TagId INT," + + " INDEX(SignerId(32), BatchId), CONSTRAINT FOREIGN KEY (TagId) REFERENCES TagTable(TagId))"); return list; } @@ -138,6 +259,8 @@ public class MySQLQueryProvider implements SQLQueryProvider { public List getSchemaDeletionCommands() { List list = new LinkedList(); + list.add("DROP TABLE IF EXISTS BatchTagTable"); + list.add("DROP TABLE IF EXISTS BatchTable"); list.add("DROP TABLE IF EXISTS MsgTagTable"); list.add("DROP TABLE IF EXISTS SignatureTable"); list.add("DROP TABLE IF EXISTS TagTable"); diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java index 945ae47..b581b87 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java @@ -54,6 +54,8 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi return "MsgTable.EntryNum = :EntryNum" + serialString; case MAX_ENTRY: return "MsgTable.EntryNum <= :EntryNum" + serialString; + case MIN_ENTRY: + return "MsgTable.EntryNum <= :EntryNum" + serialString; case MAX_MESSAGES: return "LIMIT = :Limit" + serialString; case MSG_ID: @@ -73,15 +75,32 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi @Override public String getConditionParamTypeName(FilterType filterType) throws IllegalArgumentException { - return null; //TODO: write this. + + switch(filterType) { + case EXACT_ENTRY: // Go through + case MAX_ENTRY: // Go through + case MIN_ENTRY: // Go through + case MAX_MESSAGES: + return "INTEGER"; + + case MSG_ID: // Go through + case SIGNER_ID: + return "BLOB"; + + case TAG: + return "VARCHAR"; + + default: + throw new IllegalArgumentException("Cannot serve a filter of type " + filterType); + } + } @Override public DataSource getDataSource() { - // TODO: Fix this + SQLiteDataSource dataSource = new SQLiteDataSource(); dataSource.setUrl("jdbc:sqlite:" + dbName); - dataSource.setDatabaseName("meerkat"); //TODO: Make generic return dataSource; } @@ -98,10 +117,9 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi + " REFERENCES MsgTable(EntryNum), FOREIGN KEY (TagId) REFERENCES TagTable(TagId), UNIQUE (EntryNum, TagID))"); list.add("CREATE TABLE IF NOT EXISTS SignatureTable (EntryNum INTEGER, SignerId BLOB, Signature BLOB," - + " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))"); + + " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum), UNIQUE(SignerId, EntryNum))"); list.add("CREATE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId)"); - list.add("CREATE UNIQUE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId, EntryNum)"); return list; } diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataCallbackHandler.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataCallbackHandler.java new file mode 100644 index 0000000..9ad0dc7 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataCallbackHandler.java @@ -0,0 +1,32 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import com.google.protobuf.InvalidProtocolBufferException; +import meerkat.comm.MessageOutputStream; +import meerkat.protobuf.BulletinBoardAPI.BatchData; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.jdbc.core.RowMapper; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by Arbel Deutsch Peled on 19-Dec-15. + */ +public class BatchDataCallbackHandler implements RowCallbackHandler { + + private final MessageOutputStream out; + + public BatchDataCallbackHandler(MessageOutputStream out) { + this.out = out; + } + + @Override + public void processRow(ResultSet rs) throws SQLException { + try { + out.writeMessage(BatchData.parseFrom(rs.getBytes(1))); + } catch (IOException e) { + //TODO: Log + } + } +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java new file mode 100644 index 0000000..bc4ea26 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java @@ -0,0 +1,26 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import meerkat.protobuf.BulletinBoardAPI.BatchData; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by Arbel Deutsch Peled on 19-Dec-15. + */ +public class BatchDataMapper implements RowMapper { + + @Override + public BatchData mapRow(ResultSet rs, int rowNum) throws SQLException { + + try { + return BatchData.parseFrom(rs.getBytes(1)); + } catch (InvalidProtocolBufferException e) { + return BatchData.getDefaultInstance(); + } + + } +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/EntryNumMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/LongMapper.java similarity index 87% rename from bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/EntryNumMapper.java rename to bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/LongMapper.java index 478c39e..1ec0d98 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/EntryNumMapper.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/LongMapper.java @@ -9,7 +9,7 @@ import java.sql.SQLException; /** * Created by Arbel Deutsch Peled on 11-Dec-15. */ -public class EntryNumMapper implements RowMapper { +public class LongMapper implements RowMapper { @Override public Long mapRow(ResultSet rs, int rowNum) throws SQLException { diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageCallbackHandler.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageCallbackHandler.java new file mode 100644 index 0000000..bdba241 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageCallbackHandler.java @@ -0,0 +1,80 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import com.google.protobuf.InvalidProtocolBufferException; +import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.*; +import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider.*; +import meerkat.comm.MessageOutputStream; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Crypto; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + */ +public class MessageCallbackHandler implements RowCallbackHandler { + + NamedParameterJdbcTemplate jdbcTemplate; + SQLQueryProvider sqlQueryProvider; + MessageOutputStream out; + + public MessageCallbackHandler(NamedParameterJdbcTemplate jdbcTemplate, SQLQueryProvider sqlQueryProvider, MessageOutputStream out) { + + this.jdbcTemplate = jdbcTemplate; + this.sqlQueryProvider = sqlQueryProvider; + this.out = out; + + } + + @Override + public void processRow(ResultSet rs) throws SQLException { + + BulletinBoardMessage.Builder result; + + try { + + result = BulletinBoardMessage.newBuilder() + .setEntryNum(rs.getLong(1)) + .setMsg(UnsignedBulletinBoardMessage.parseFrom(rs.getBytes(2))); + + + } catch (InvalidProtocolBufferException e) { + //TODO: log + return; + } + + // Retrieve signatures + + MapSqlParameterSource sqlParameterSource = new MapSqlParameterSource(); + sqlParameterSource.addValue(QueryType.GET_SIGNATURES.getParamName(0), result.getEntryNum()); + + List signatures = jdbcTemplate.query( + sqlQueryProvider.getSQLString(QueryType.GET_SIGNATURES), + sqlParameterSource, + new SignatureMapper()); + + // Append signatures + result.addAllSig(signatures); + + // Finalize message and add to message list. + + try { + + out.writeMessage(result.build()); + + } catch (IOException e) { + + //TODO: log + e.printStackTrace(); + + } + + } + +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubMapper.java new file mode 100644 index 0000000..1f9c459 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubMapper.java @@ -0,0 +1,31 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; +import meerkat.protobuf.BulletinBoardAPI.UnsignedBulletinBoardMessage; +import meerkat.util.BulletinBoardUtils; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by Arbel Deutsch Peled on 11-Dec-15. + */ +public class MessageStubMapper implements RowMapper { + + @Override + public BulletinBoardMessage mapRow(ResultSet rs, int rowNum) throws SQLException { + + return BulletinBoardMessage.newBuilder() + .setEntryNum(rs.getLong(1)) + .setMsg(UnsignedBulletinBoardMessage.newBuilder() + .setData(ByteString.copyFrom(rs.getBytes(2))) + .setTimestamp(BulletinBoardUtils.toTimestampProto(rs.getTimestamp(3))) + .build()) + .build(); + + } + +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/StringMapper.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/StringMapper.java new file mode 100644 index 0000000..c5b1d85 --- /dev/null +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/StringMapper.java @@ -0,0 +1,18 @@ +package meerkat.bulletinboard.sqlserver.mappers; + +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by Arbel Deutsch Peled on 20-Dec-15. + */ +public class StringMapper implements RowMapper { + + @Override + public String mapRow(ResultSet rs, int rowNum) throws SQLException { + return rs.getString(1); + } + +} diff --git a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java index b3fc03c..2e4782f 100644 --- a/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java +++ b/bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java @@ -3,13 +3,10 @@ package meerkat.bulletinboard.webapp; import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.StreamingOutput; import meerkat.bulletinboard.BulletinBoardServer; import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer; @@ -17,13 +14,20 @@ import meerkat.bulletinboard.sqlserver.H2QueryProvider; import meerkat.bulletinboard.sqlserver.MySQLQueryProvider; import meerkat.bulletinboard.sqlserver.SQLiteQueryProvider; import meerkat.comm.CommunicationException; -import meerkat.protobuf.BulletinBoardAPI.BoolMsg; -import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; -import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessageList; -import meerkat.protobuf.BulletinBoardAPI.MessageFilterList; -import meerkat.rest.Constants; +import meerkat.comm.MessageOutputStream; +import meerkat.protobuf.BulletinBoardAPI; +import meerkat.protobuf.BulletinBoardAPI.*; +import static meerkat.bulletinboard.BulletinBoardConstants.*; +import static meerkat.rest.Constants.*; -@Path(Constants.BULLETIN_BOARD_SERVER_PATH) +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +/** + * An implementation of the BulletinBoardServer which functions as a WebApp + */ +@Path(BULLETIN_BOARD_SERVER_PATH) public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextListener{ private static final String BULLETIN_BOARD_ATTRIBUTE_NAME = "bulletinBoard"; @@ -79,24 +83,146 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL } } - @Path(Constants.POST_MESSAGE_PATH) + @Path(POST_MESSAGE_PATH) @POST - @Consumes(Constants.MEDIATYPE_PROTOBUF) - @Produces(Constants.MEDIATYPE_PROTOBUF) + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) @Override public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException { init(); return bulletinBoard.postMessage(msg); } - - @Path(Constants.READ_MESSAGES_PATH) - @POST - @Consumes(Constants.MEDIATYPE_PROTOBUF) - @Produces(Constants.MEDIATYPE_PROTOBUF) + @Override - public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException { + public void readMessages(MessageFilterList filterList, MessageOutputStream out) throws CommunicationException { init(); - return bulletinBoard.readMessages(filterList); + bulletinBoard.readMessages(filterList, out); + } + + + @Path(READ_MESSAGES_PATH) + @POST + @Consumes(MEDIATYPE_PROTOBUF) + /** + * Wrapper for the readMessages method which streams the output into the response + */ + public StreamingOutput readMessages(final MessageFilterList filterList) { + + return new StreamingOutput() { + + @Override + public void write(OutputStream output) throws IOException, WebApplicationException { + MessageOutputStream out = new MessageOutputStream<>(output); + + try { + init(); + bulletinBoard.readMessages(filterList, out); + } catch (CommunicationException e) { + //TODO: Log + out.writeMessage(null); + } + } + + }; + + } + + @Path(BEGIN_BATCH_PATH) + @POST + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) + @Override + public BoolMsg beginBatch(BeginBatchMessage message) { + try { + init(); + return bulletinBoard.beginBatch(message); + } catch (CommunicationException e) { + System.err.println(e.getMessage()); + return null; + } + } + + @Path(POST_BATCH_PATH) + @POST + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) + @Override + public BoolMsg postBatchMessage(BatchMessage batchMessage) { + try { + init(); + return bulletinBoard.postBatchMessage(batchMessage); + } catch (CommunicationException e) { + System.err.println(e.getMessage()); + return null; + } + } + + @Path(CLOSE_BATCH_PATH) + @POST + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) + @Override + public BoolMsg closeBatchMessage(CloseBatchMessage message) { + try { + init(); + return bulletinBoard.closeBatchMessage(message); + } catch (CommunicationException e) { + System.err.println(e.getMessage()); + return null; + } + } + + + @Override + public void readBatch(BatchSpecificationMessage message, MessageOutputStream out) { + try { + init(); + bulletinBoard.readBatch(message, out); + } catch (CommunicationException | IllegalArgumentException e) { + System.err.println(e.getMessage()); + } + } + + @Path(READ_BATCH_PATH) + @POST + @Consumes(MEDIATYPE_PROTOBUF) + /** + * Wrapper for the readBatch method which streams the output into the response + */ + public StreamingOutput readBatch(final BatchSpecificationMessage message) { + + return new StreamingOutput() { + + @Override + public void write(OutputStream output) throws IOException, WebApplicationException { + MessageOutputStream out = new MessageOutputStream<>(output); + + try { + init(); + bulletinBoard.readBatch(message, out); + } catch (CommunicationException e) { + //TODO: Log + out.writeMessage(null); + } + } + + }; + + } + + @Path(SYNC_QUERY_PATH) + @POST + @Consumes(MEDIATYPE_PROTOBUF) + @Produces(MEDIATYPE_PROTOBUF) + @Override + public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException { + try{ + init(); + return bulletinBoard.querySync(syncQuery); + } catch (CommunicationException | IllegalArgumentException e) { + System.err.println(e.getMessage()); + return null; + } } @Override diff --git a/bulletin-board-server/src/main/webapp/WEB-INF/web.xml b/bulletin-board-server/src/main/webapp/WEB-INF/web.xml index 2198c07..226aa3b 100644 --- a/bulletin-board-server/src/main/webapp/WEB-INF/web.xml +++ b/bulletin-board-server/src/main/webapp/WEB-INF/web.xml @@ -31,7 +31,7 @@ mypass dbType - SQLite + H2 meerkat.bulletinboard.webapp.BulletinBoardWebApp diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java index 838adcc..744a086 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java @@ -4,8 +4,11 @@ package meerkat.bulletinboard; import com.google.protobuf.ByteString; import com.google.protobuf.TextFormat; +import com.google.protobuf.Timestamp; +import meerkat.comm.MessageInputStream; import meerkat.protobuf.Crypto.*; import meerkat.protobuf.BulletinBoardAPI.*; +import static meerkat.bulletinboard.BulletinBoardConstants.*; import meerkat.rest.Constants; import meerkat.rest.ProtobufMessageBodyReader; import meerkat.rest.ProtobufMessageBodyWriter; @@ -17,7 +20,10 @@ import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.io.InputStream; +import java.util.List; public class BulletinBoardSQLServerIntegrationTest { @@ -43,6 +49,16 @@ public class BulletinBoardSQLServerIntegrationTest { byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24}; byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0}; + Timestamp t1 = Timestamp.newBuilder() + .setSeconds(8276482) + .setNanos(4314) + .build(); + + Timestamp t2 = Timestamp.newBuilder() + .setSeconds(987591) + .setNanos(1513) + .build(); + WebTarget webTarget; Response response; BoolMsg bool; @@ -50,12 +66,12 @@ public class BulletinBoardSQLServerIntegrationTest { BulletinBoardMessage msg; MessageFilterList filterList; - BulletinBoardMessageList msgList; + List msgList; // Test writing mechanism - System.err.println("******** Testing: " + Constants.POST_MESSAGE_PATH); - webTarget = client.target(BASE_URL).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.POST_MESSAGE_PATH); + System.err.println("******** Testing: " + POST_MESSAGE_PATH); + webTarget = client.target(BASE_URL).path(BULLETIN_BOARD_SERVER_PATH).path(POST_MESSAGE_PATH); System.err.println(webTarget.getUri()); msg = BulletinBoardMessage.newBuilder() @@ -63,6 +79,7 @@ public class BulletinBoardSQLServerIntegrationTest { .addTag("Signature") .addTag("Trustee") .setData(ByteString.copyFrom(b1)) + .setTimestamp(t1) .build()) .addSig(Signature.newBuilder() .setType(SignatureType.DSA) @@ -86,6 +103,7 @@ public class BulletinBoardSQLServerIntegrationTest { .addTag("Vote") .addTag("Trustee") .setData(ByteString.copyFrom(b4)) + .setTimestamp(t2) .build()) .addSig(Signature.newBuilder() .setType(SignatureType.ECDSA) @@ -101,8 +119,8 @@ public class BulletinBoardSQLServerIntegrationTest { // Test reading mechanism - System.err.println("******** Testing: " + Constants.READ_MESSAGES_PATH); - webTarget = client.target(BASE_URL).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH); + System.err.println("******** Testing: " + READ_MESSAGES_PATH); + webTarget = client.target(BASE_URL).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH); filterList = MessageFilterList.newBuilder() .addFilter( MessageFilter.newBuilder() @@ -112,13 +130,20 @@ public class BulletinBoardSQLServerIntegrationTest { ) .build(); - response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF)); - System.err.println(response); - msgList = response.readEntity(BulletinBoardMessageList.class); - System.err.println("List size: " + msgList.getMessageCount()); + InputStream in = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF), InputStream.class); + + MessageInputStream inputStream = + MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BulletinBoardMessage.class); + + msgList = inputStream.asList(); + System.err.println("List size: " + msgList.size()); System.err.println("This is the list:"); - System.err.println(TextFormat.printToString(msgList)); - assert msgList.getMessageCount() == 1; + + for (BulletinBoardMessage message : msgList) { + System.err.println(TextFormat.printToString(message)); + } + + assert msgList.size() == 1; } } diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java index 4799e0d..e33c739 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java @@ -1,9 +1,12 @@ package meerkat.bulletinboard; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; +import java.lang.reflect.InvocationTargetException; import java.math.BigInteger; import java.security.InvalidKeyException; import java.security.KeyStore; @@ -12,26 +15,27 @@ import java.security.NoSuchAlgorithmException; import java.security.SignatureException; import java.security.UnrecoverableKeyException; import java.security.cert.CertificateException; -import java.util.List; -import java.util.Random; +import java.util.*; import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; import meerkat.comm.CommunicationException; +import meerkat.comm.MessageInputStream; +import meerkat.comm.MessageOutputStream; +import meerkat.comm.MessageInputStream.MessageInputStreamFactory; import meerkat.crypto.concrete.ECDSASignature; -import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage; -import meerkat.protobuf.BulletinBoardAPI.FilterType; -import meerkat.protobuf.BulletinBoardAPI.MessageFilter; -import meerkat.protobuf.BulletinBoardAPI.MessageFilterList; -import meerkat.protobuf.BulletinBoardAPI.UnsignedBulletinBoardMessage; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.util.BulletinBoardUtils; import static org.junit.Assert.*; import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.MatcherAssert.assertThat; public class GenericBulletinBoardServerTest { protected BulletinBoardServer bulletinBoardServer; - private ECDSASignature signers[]; + private GenericBatchDigitalSignature[] signers; private ByteString[] signerIDs; private Random random; @@ -51,6 +55,8 @@ public class GenericBulletinBoardServerTest { private String[] tags; private byte[][] data; + private List completeBatches; + private final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); // Used to time the tests /** @@ -71,10 +77,10 @@ public class GenericBulletinBoardServerTest { this.bulletinBoardServer = bulletinBoardServer; - signers = new ECDSASignature[2]; + signers = new GenericBatchDigitalSignature[2]; signerIDs = new ByteString[signers.length]; - signers[0] = new ECDSASignature(); - signers[1] = new ECDSASignature(); + signers[0] = new GenericBatchDigitalSignature(new ECDSASignature()); + signers[1] = new GenericBatchDigitalSignature(new ECDSASignature()); InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE); char[] password = KEYFILE_PASSWORD1.toCharArray(); @@ -121,6 +127,11 @@ public class GenericBulletinBoardServerTest { long end = threadBean.getCurrentThreadCpuTime(); System.err.println("Finished initializing GenericBulletinBoardServerTest"); System.err.println("Time of operation: " + (end - start)); + + // Initialize Batch variables + + completeBatches = new ArrayList(10); + } private byte randomByte(){ @@ -170,7 +181,8 @@ public class GenericBulletinBoardServerTest { for (i = 1; i <= MESSAGE_NUM; i++) { unsignedMsgBuilder = UnsignedBulletinBoardMessage.newBuilder() - .setData(ByteString.copyFrom(data[i - 1])); + .setData(ByteString.copyFrom(data[i - 1])) + .setTimestamp(BulletinBoardUtils.toTimestampProto()); // Add tags based on bit-representation of message number. @@ -230,28 +242,39 @@ public class GenericBulletinBoardServerTest { System.err.println("Starting to test tag and signature mechanism"); long start = threadBean.getCurrentThreadCpuTime(); - List messages; + List messages = new LinkedList<>(); // Check tag mechanism - + for (int i = 0 ; i < TAG_NUM ; i++){ // Retrieve messages having tag i try { - messages = bulletinBoardServer.readMessages( - MessageFilterList.newBuilder() - .addFilter(MessageFilter.newBuilder() - .setType(FilterType.TAG) - .setTag(tags[i]) - .build() - ) - .build() - ) - .getMessageList(); + MessageFilterList filterList = MessageFilterList.newBuilder() + .addFilter(MessageFilter.newBuilder() + .setType(FilterType.TAG) + .setTag(tags[i]) + .build() + ) + .build(); - } catch (CommunicationException e) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + bulletinBoardServer.readMessages(filterList, new MessageOutputStream(outputStream)); + + MessageInputStream inputStream = + MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream( + outputStream.toByteArray()), + BulletinBoardMessage.class); + + messages = inputStream.asList(); + + } catch (CommunicationException | IOException e) { + fail(e.getMessage()); + return; + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { fail(e.getMessage()); return; } @@ -328,11 +351,26 @@ public class GenericBulletinBoardServerTest { ); try { - messages = bulletinBoardServer.readMessages(filterListBuilder.build()).getMessageList(); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + bulletinBoardServer.readMessages(filterListBuilder.build(), new MessageOutputStream(outputStream)); + + MessageInputStream inputStream = + MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream( + outputStream.toByteArray()), + BulletinBoardMessage.class); + + messages = inputStream.asList(); + } catch (CommunicationException e) { System.err.println("Failed retrieving multi-tag messages from DB: " + e.getMessage()); fail("Failed retrieving multi-tag messages from DB: " + e.getMessage()); return; + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | IOException e) { + System.err.println("Falied to read from stream while retrieving multi-tag messages: " + e.getMessage()); + fail("Falied to read from stream while retrieving multi-tag messages: " + e.getMessage()); + return; } expectedMsgCount /= 2; @@ -359,11 +397,26 @@ public class GenericBulletinBoardServerTest { .build()); try { - messages = bulletinBoardServer.readMessages(filterListBuilder.build()).getMessageList(); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + bulletinBoardServer.readMessages(filterListBuilder.build(), new MessageOutputStream(outputStream)); + + MessageInputStream inputStream = + MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream( + outputStream.toByteArray()), + BulletinBoardMessage.class); + + messages = inputStream.asList(); + } catch (CommunicationException e) { System.err.println("Failed retrieving multi-signature message from DB: " + e.getMessage()); fail("Failed retrieving multi-signature message from DB: " + e.getMessage()); return; + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | IOException e) { + System.err.println("Falied to read from stream while retrieving multi-signature message: " + e.getMessage()); + fail("Falied to read from stream while retrieving multi-signature message: " + e.getMessage()); + return; } assertThat(messages.size(), is(MESSAGE_NUM / 4)); @@ -377,6 +430,195 @@ public class GenericBulletinBoardServerTest { System.err.println("Time of operation: " + (end - start)); } + + /** + * Tests that posting a message before opening a batch does not work + * @throws CommunicationException + */ + public void testBatchPostAfterClose() throws CommunicationException, SignatureException { + + final int BATCH_ID = 100; + + CompleteBatch completeBatch = new CompleteBatch(Timestamp.newBuilder() + .setSeconds(978325) + .setNanos(8097234) + .build()); + BoolMsg result; + + // Create data + + completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder() + .setSignerId(signerIDs[1]) + .setBatchId(BATCH_ID) + .addTag("Test") + .build()); + + BatchData batchData = BatchData.newBuilder() + .setData(ByteString.copyFrom((new byte[] {1,2,3,4}))) + .build(); + + completeBatch.appendBatchData(batchData); + + signers[1].updateContent(completeBatch); + + completeBatch.setSignature(signers[1].sign()); + + // Begin batch + + result = bulletinBoardServer.beginBatch(completeBatch.getBeginBatchMessage()); + + assertThat("Was not able to open batch", result.getValue(), is(true)); + + // Post data + + BatchMessage batchMessage = BatchMessage.newBuilder() + .setSignerId(signerIDs[1]) + .setBatchId(BATCH_ID) + .setData(batchData) + .build(); + + result = bulletinBoardServer.postBatchMessage(batchMessage); + + assertThat("Was not able to post batch message", result.getValue(), is(true)); + + // Close batch + + result = bulletinBoardServer.closeBatchMessage(completeBatch.getCloseBatchMessage()); + + assertThat("Was not able to close batch", result.getValue(), is(true)); + + // Attempt to open batch again + + result = bulletinBoardServer.beginBatch(completeBatch.getBeginBatchMessage()); + + assertThat("Was able to open a closed batch", result.getValue(), is(false)); + + // Attempt to add batch data + + result = bulletinBoardServer.postBatchMessage(batchMessage); + + assertThat("Was able to change a closed batch", result.getValue(), is(false)); + + } + + /** + * Posts a complete batch message + * @throws CommunicationException + */ + public void testPostBatch() throws CommunicationException, SignatureException { + + CompleteBatch completeBatch = new CompleteBatch(Timestamp.newBuilder() + .setSeconds(12345) + .setNanos(1111) + .build()); + int currentBatch = completeBatches.size(); + + BoolMsg result; + + // Define batch data + + String[] tempBatchTags = new String[]{randomString(),randomString(),randomString()}; + byte[][] tempBatchData = new byte[Math.abs(randomByte())][]; + + for (int i = 0 ; i < tempBatchData.length ; i++) { + + tempBatchData[i] = new byte[Math.abs(randomByte())]; + + for (int j = 0; j < tempBatchData[i].length; j++) { + tempBatchData[i][j] = randomByte(); + } + + } + + // Begin batch + + completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder() + .setSignerId(signerIDs[0]) + .setBatchId(currentBatch) + .addAllTag(Arrays.asList(tempBatchTags)) + .build()); + + result = bulletinBoardServer.beginBatch(completeBatch.getBeginBatchMessage()); + + assertThat("Could not begin batch " + currentBatch, result.getValue(), is(true)); + + // Add batch data and randomize data posting order + + List dataOrder = new ArrayList(tempBatchData.length); + for (int i = 0 ; i < tempBatchData.length ; i++) { + dataOrder.add(i); + completeBatch.appendBatchData(BatchData.newBuilder() + .setData(ByteString.copyFrom(tempBatchData[i])) + .build()); + } + Collections.shuffle(dataOrder); + + // Post data + + for (int i = 0 ; i < tempBatchData.length ; i++) { + + int dataIndex = dataOrder.get(i); + + result = bulletinBoardServer.postBatchMessage(BatchMessage.newBuilder() + .setSignerId(signerIDs[0]) + .setBatchId(currentBatch) + .setSerialNum(dataIndex) + .setData(completeBatch.getBatchDataList().get(dataIndex)) + .build()); + + assertThat("Could not post batch data for batch ID " + currentBatch + " serial number " + dataIndex, + result.getValue(), is(true)); + + } + + // Close batch + + signers[0].updateContent(completeBatch); + completeBatch.setSignature(signers[0].sign()); + + result = bulletinBoardServer.closeBatchMessage(completeBatch.getCloseBatchMessage()); + + assertThat("Could not close batch " + currentBatch, result.getValue(), is(true)); + + // Update locally stored batches + completeBatches.add(completeBatch); + + } + + public void testReadBatch() throws CommunicationException { + + for (CompleteBatch completeBatch : completeBatches) { + + try { + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + BatchSpecificationMessage batchSpecificationMessage = + BatchSpecificationMessage.newBuilder() + .setSignerId(completeBatch.getBeginBatchMessage().getSignerId()) + .setBatchId(completeBatch.getBeginBatchMessage().getBatchId()) + .setStartPosition(0) + .build(); + + bulletinBoardServer.readBatch(batchSpecificationMessage, new MessageOutputStream(outputStream)); + + MessageInputStream inputStream = + MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream( + outputStream.toByteArray()), + BatchData.class); + + List batchDataList = inputStream.asList(); + + assertThat("Non-matching batch data for batch " + completeBatch.getBeginBatchMessage().getBatchId(), + completeBatch.getBatchDataList().equals(batchDataList), is(true)); + + } catch (IOException | NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + assertThat("Error reading batch data list from input stream", false); + } + + } + + } public void close(){ signers[0].clearSigningKey(); diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java index ef19310..577c9be 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java @@ -107,6 +107,39 @@ public class H2BulletinBoardServerTest { System.err.println("Time of operation: " + (end - start)); } + @Test + public void testBatchPostAfterClose() { + try{ + serverTest.testBatchPostAfterClose(); + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + } + + @Test + public void testBatch() { + + final int BATCH_NUM = 20; + + try{ + for (int i = 0 ; i < BATCH_NUM ; i++) { + serverTest.testPostBatch(); + } + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + + try{ + serverTest.testReadBatch(); + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + + } + @After public void close() { System.err.println("Starting to close H2BulletinBoardServerTest"); diff --git a/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java b/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java index e473931..42c11f7 100644 --- a/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java +++ b/bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java @@ -111,6 +111,39 @@ public class MySQLBulletinBoardServerTest { System.err.println("Time of operation: " + (end - start)); } + @Test + public void testBatchPostAfterClose() { + try{ + serverTest.testBatchPostAfterClose(); + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + } + + @Test + public void testBatch() { + + final int BATCH_NUM = 20; + + try{ + for (int i = 0 ; i < BATCH_NUM ; i++) { + serverTest.testPostBatch(); + } + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + + try{ + serverTest.testReadBatch(); + } catch (Exception e) { + System.err.println(e.getMessage()); + fail(e.getMessage()); + } + + } + @After public void close() { System.err.println("Starting to close MySQLBulletinBoardServerTest"); diff --git a/meerkat-common/build.gradle b/meerkat-common/build.gradle index c510e0a..a9035b0 100644 --- a/meerkat-common/build.gradle +++ b/meerkat-common/build.gradle @@ -39,6 +39,7 @@ version += "${isSnapshot ? '-SNAPSHOT' : ''}" dependencies { // Logging compile 'org.slf4j:slf4j-api:1.7.7' + compile 'javax.ws.rs:javax.ws.rs-api:2.0.+' runtime 'ch.qos.logback:logback-classic:1.1.2' runtime 'ch.qos.logback:logback-core:1.1.2' @@ -46,7 +47,7 @@ dependencies { compile 'com.google.protobuf:protobuf-java:3.+' // ListeningExecutor - compile 'com.google.guava:guava:11.0.+' + compile 'com.google.guava:guava:15.0' // Crypto compile 'org.factcenter.qilin:qilin:1.2+' diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java new file mode 100644 index 0000000..c6e330c --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java @@ -0,0 +1,110 @@ +package meerkat.bulletinboard; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.protobuf.ByteString; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 14-Dec-15. + */ +public interface AsyncBulletinBoardClient extends BulletinBoardClient { + + public interface MessageHandler { + void handleNewMessages(List messageList); + } + + /** + * Post a message to the bulletin board in an asynchronous manner + * @param msg is the message to be posted + * @param callback is a class containing methods to handle the result of the operation + * @return a unique message ID for the message, that can be later used to retrieve the batch + */ + public MessageID postMessage(BulletinBoardMessage msg, FutureCallback callback); + + /** + * Perform an end-to-end post of a signed batch message + * @param completeBatch contains all the data of the batch including the meta-data and the signature + * @param callback is a class containing methods to handle the result of the operation + * @return a unique identifier for the batch message + */ + public MessageID postBatch(CompleteBatch completeBatch, FutureCallback callback); + + /** + * This message informs the server about the existence of a new batch message and supplies it with the tags associated with it + * @param beginBatchMessage contains the data required to begin the batch + * @param callback is a callback function class for handling results of the operation + */ + public void beginBatch(BeginBatchMessage beginBatchMessage, FutureCallback callback); + + /** + * This method posts batch data into an (assumed to be open) batch + * It does not close the batch + * @param signerId is the canonical form for the ID of the sender of this batch + * @param batchId is a unique (per signer) ID for this batch + * @param batchDataList is the (canonically ordered) list of data comprising the portion of the batch to be posted + * @param startPosition is the location (in the batch) of the first entry in batchDataList + * (optionally used to continue interrupted post operations) + * The first position in the batch is position 0 + * @param callback is a callback function class for handling results of the operation + */ + public void postBatchData(byte[] signerId, int batchId, List batchDataList, + int startPosition, FutureCallback callback); + + /** + * Overloading of the postBatchData method which starts at the first position in the batch + */ + public void postBatchData(byte[] signerId, int batchId, List batchDataList, FutureCallback callback); + + /** + * Overloading of the postBatchData method which uses ByteString + */ + public void postBatchData(ByteString signerId, int batchId, List batchDataList, + int startPosition, FutureCallback callback); + + /** + * Overloading of the postBatchData method which uses ByteString and starts at the first position in the batch + */ + public void postBatchData(ByteString signerId, int batchId, List batchDataList, FutureCallback callback); + + /** + * Attempts to close a batch message + * @param closeBatchMessage contains the data required to close the batch + * @param callback is a callback function class for handling results of the operation + */ + public void closeBatch(CloseBatchMessage closeBatchMessage, FutureCallback callback); + + /** + * Check how "safe" a given message is in an asynchronous manner + * The result of the computation is a rank between 0.0 and 1.0 indicating the fraction of servers containing the message + * @param id is the unique message identifier for retrieval + * @param callback is a callback function class for handling results of the operation + */ + public void getRedundancy(MessageID id, FutureCallback callback); + + /** + * Read all messages posted matching the given filter in an asynchronous manner + * Note that if messages haven't been "fully posted", this might return a different + * set of messages in different calls. However, messages that are fully posted + * are guaranteed to be included. + * @param filterList return only messages that match the filters (null means no filtering). + * @param callback is a callback function class for handling results of the operation + */ + public void readMessages(MessageFilterList filterList, FutureCallback> callback); + + /** + * Read a given batch message from the bulletin board + * @param batchSpecificationMessage contains the data required to specify a single batch instance + * @param callback is a callback class for handling the result of the operation + */ + public void readBatch(BatchSpecificationMessage batchSpecificationMessage, FutureCallback callback); + + /** + * Subscribes to a notifier that will return any new messages on the server that match the given filters + * @param filterList defines the set of filters for message retrieval + * @param messageHandler defines the handler for new messages received + */ + public void subscribe(MessageFilterList filterList, MessageHandler messageHandler); + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigest.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigest.java new file mode 100644 index 0000000..6e30fe9 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigest.java @@ -0,0 +1,20 @@ +package meerkat.bulletinboard; + +import meerkat.crypto.Digest; +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 18-Dec-15. + * Extends the Digest interface with a method for digesting Batch messages + */ +public interface BatchDigest extends Digest { + + /** + * Update the digest with the batch message data (ignore the signature) + * @param completeBatch is the batch message that needs to be digested + */ + public void update(CompleteBatch completeBatch); + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigitalSignature.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigitalSignature.java new file mode 100644 index 0000000..e04f3c5 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigitalSignature.java @@ -0,0 +1,34 @@ +package meerkat.bulletinboard; + +import meerkat.crypto.DigitalSignature; +import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage; +import meerkat.protobuf.BulletinBoardAPI.BatchData; +import meerkat.protobuf.Crypto.Signature; + +import java.security.InvalidKeyException; +import java.security.SignatureException; +import java.security.cert.CertificateException; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 20-Dec-15. + * Extends the DigitalSignature interface with methods for signing and authenticating Batch messages + */ +public interface BatchDigitalSignature extends DigitalSignature { + + /** + * Appends the batch data to the signed content (ignoring the signature) + * @param completeBatch contains all the data about the batch + * @throws SignatureException + */ + public void updateContent(CompleteBatch completeBatch) throws SignatureException; + + /** + * Performs a complete verification process on the given batch message + * @param completeBatch contains the batch data as well as the signature + * @return TRUE if the batch is verified and FALSE otherwise + * @throws SignatureException | SignatureException | InvalidKeyException when underlying methods do so + */ + public boolean verify(CompleteBatch completeBatch) throws SignatureException, CertificateException, InvalidKeyException; + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java index c51e561..2f5a3df 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java @@ -1,6 +1,6 @@ package meerkat.bulletinboard; -import meerkat.comm.*; +import meerkat.comm.CommunicationException; import meerkat.protobuf.Voting.*; import static meerkat.protobuf.BulletinBoardAPI.*; @@ -12,11 +12,6 @@ import java.util.List; */ public interface BulletinBoardClient { - interface ClientCallback { - void handleCallback(T msg); - void handleFailure(Throwable t); - } - /** * Initialize the client to use some specified servers * @param clientParams contains the parameters required for the client setup @@ -24,26 +19,31 @@ public interface BulletinBoardClient { void init(BulletinBoardClientParams clientParams); /** - * Post a message to the bulletin board - * @param msg + * Post a message to the bulletin board in a synchronous manner + * @param msg is the message to be posted + * @return a unique message ID for the message, that can be later used to retrieve the batch + * @throws CommunicationException */ - MessageID postMessage(BulletinBoardMessage msg, ClientCallback callback); + MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException; + + /** - * Check how "safe" a given message is - * @param id + * Check how "safe" a given message is in a synchronous manner + * @param id is the unique message identifier for retrieval * @return a normalized "redundancy score" from 0 (local only) to 1 (fully published) */ - void getRedundancy(MessageID id, ClientCallback callback); + float getRedundancy(MessageID id); /** - * Read all messages posted matching the given filter + * Read all messages posted matching the given filter in a synchronous manner * Note that if messages haven't been "fully posted", this might return a different * set of messages in different calls. However, messages that are fully posted * are guaranteed to be included. * @param filterList return only messages that match the filters (null means no filtering). + * @return the list of messages */ - void readMessages(MessageFilterList filterList, ClientCallback> callback); + List readMessages(MessageFilterList filterList); /** * Closes all connections, if any. diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java new file mode 100644 index 0000000..66652f8 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java @@ -0,0 +1,24 @@ +package meerkat.bulletinboard; + +/** + * Created by Arbel Deutsch Peled on 21-Dec-15. + */ +public interface BulletinBoardConstants { + + // Relative addresses for Bulletin Board operations + + public static final String BULLETIN_BOARD_SERVER_PATH = "/bbserver"; + public static final String READ_MESSAGES_PATH = "/readmessages"; + public static final String READ_BATCH_PATH = "/readbatch"; + public static final String POST_MESSAGE_PATH = "/postmessage"; + public static final String BEGIN_BATCH_PATH = "/beginbatch"; + public static final String POST_BATCH_PATH = "/postbatch"; + public static final String CLOSE_BATCH_PATH = "/closebatch"; + public static final String SYNC_QUERY_PATH = "/syncquery"; + + // Other Constants + + public static final String BATCH_TAG = "@BATCH"; + public static final String BATCH_ID_TAG_PREFIX = "BATCHID#"; + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java index da53c1f..0b279c2 100644 --- a/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java @@ -1,42 +1,94 @@ package meerkat.bulletinboard; import meerkat.comm.CommunicationException; +import meerkat.comm.MessageOutputStream; import meerkat.protobuf.BulletinBoardAPI.*; + /** * Created by Arbel on 07/11/15. * - * This interface refers to a single instance of a Bulletin Board. - * An implementation of this interface may use any DB and be hosted on any machine. + * This interface refers to a single instance of a Bulletin Board + * An implementation of this interface may use any DB and be hosted on any machine. */ public interface BulletinBoardServer{ /** - * This method initializes the server by reading the signature data and storing it. - * It also establishes the connection to the DB. - * @throws CommunicationException on DB connection error. + * This method initializes the server by reading the signature data and storing it + * It also establishes the connection to the DB + * @throws CommunicationException on DB connection error */ public void init(String meerkatDB) throws CommunicationException; /** * Post a message to bulletin board. * @param msg is the actual (signed) message - * @return TRUE if the message has been authenticated and FALSE otherwise (in ProtoBuf form). - * @throws CommunicationException on DB connection error. + * @return TRUE if the message has been authenticated and FALSE otherwise (in ProtoBuf form) + * @throws CommunicationException on DB connection error */ - public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException; + public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException; /** - * Read all messages posted matching the given filter. - * @param filter return only messages that match the filter (empty list means no filtering). - * @return + * Read all messages posted matching the given filter + * @param filterList return only messages that match the filters (empty list or null means no filtering) + * @param out is an output stream into which the matching messages are written + * @throws CommunicationException on DB connection error */ - BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException; - + public void readMessages(MessageFilterList filterList, MessageOutputStream out) throws CommunicationException; + + /** + * Informs server about a new batch message + * @param message contains the required data about the new batch + * @return TRUE if the batch request is accepted amd FALSE otherwise + * Specifically, if such a batch already exists and is not yet closed: the value returned will be TRUE + * However, if such a batch exists and is already closed: the value returned will be FALSE + * @throws CommunicationException on DB connection error + */ + public BoolMsg beginBatch(BeginBatchMessage message) throws CommunicationException; + + /** + * Posts a (part of a) batch message to the bulletin board + * Note that the existence and contents of a batch message are not available for reading before the batch is finalized + * @param batchMessage contains the (partial) data this message carries as well as meta-data required in order to place the data + * in the correct position inside the correct batch + * @return TRUE if the message is accepted and successfully saved and FALSE otherwise + * Specifically, if the batch is already closed: the value returned will be FALSE + * However, requiring to open a batch before insertion of messages is implementation-dependent + * @throws CommunicationException on DB connection error + */ + public BoolMsg postBatchMessage(BatchMessage batchMessage) throws CommunicationException; + + /** + * Attempts to close and finalize a batch message + * @param message contains the data necessary to close the batch; in particular: the signature for the batch + * @return TRUE if the batch was successfully closed, FALSE otherwise + * Specifically, if the signature is invalid or if some of the batch parts have not yet been submitted: the value returned will be FALSE + * @throws CommunicationException on DB connection error + */ + public BoolMsg closeBatchMessage(CloseBatchMessage message) throws CommunicationException; + + /** + * Reads a batch message from the server (starting with the supplied position) + * @param message specifies the signer ID and the batch ID to read as well as an (optional) start position + * @param out is a stream of the ordered batch messages starting from the specified start position (if given) or from the beginning (if omitted) + * @throws CommunicationException on DB connection error + * @throws IllegalArgumentException if message does not specify a batch + */ + public void readBatch(BatchSpecificationMessage message, MessageOutputStream out) throws CommunicationException, IllegalArgumentException; + + + /** + * Queries the database for sync status with respect to a given sync query + * @param syncQuery contains a succinct representation of states to compare to + * @return a SyncQueryResponse object containing the representation of the most recent state the database matches + * @throws CommunicationException + */ + public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException; + /** - * This method closes the connection to the DB. - * @throws CommunicationException on DB connection error. + * This method closes the connection to the DB + * @throws CommunicationException on DB connection error */ - public void close() throws CommunicationException; + public void close() throws CommunicationException; } diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java new file mode 100644 index 0000000..14e87e7 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java @@ -0,0 +1,145 @@ +package meerkat.bulletinboard; + +import com.google.protobuf.Timestamp; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Crypto.*; +import meerkat.util.BulletinBoardMessageComparator; + +import java.util.LinkedList; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 14-Dec-15. + * + * A data structure for holding a complete batch message along with its signature + */ +public class CompleteBatch { + + private BeginBatchMessage beginBatchMessage; + private List batchDataList; + private Signature signature; + private Timestamp timestamp; + + public CompleteBatch() { + batchDataList = new LinkedList(); + } + + public CompleteBatch(BeginBatchMessage newBeginBatchMessage) { + this(); + beginBatchMessage = newBeginBatchMessage; + } + + public CompleteBatch(BeginBatchMessage newBeginBatchMessage, List newDataList) { + this(newBeginBatchMessage); + appendBatchData(newDataList); + } + + public CompleteBatch(BeginBatchMessage newBeginBatchMessage, List newDataList, Signature newSignature) { + this(newBeginBatchMessage, newDataList); + signature = newSignature; + } + + public CompleteBatch(BeginBatchMessage newBeginBatchMessage, List newDataList, Signature newSignature, Timestamp timestamp) { + this(newBeginBatchMessage, newDataList, newSignature); + this.timestamp = timestamp; + } + + public CompleteBatch(Timestamp timestamp) { + this(); + this.timestamp = timestamp; + } + + public BeginBatchMessage getBeginBatchMessage() { + return beginBatchMessage; + } + + public List getBatchDataList() { + return batchDataList; + } + + public Signature getSignature() { + return signature; + } + + public Timestamp getTimestamp() { + return timestamp; + } + + public CloseBatchMessage getCloseBatchMessage() { + return CloseBatchMessage.newBuilder() + .setBatchId(getBeginBatchMessage().getBatchId()) + .setBatchLength(getBatchDataList().size()) + .setSig(getSignature()) + .setTimestamp(getTimestamp()) + .build(); + } + + public void setBeginBatchMessage(BeginBatchMessage beginBatchMessage) { + this.beginBatchMessage = beginBatchMessage; + } + + public void appendBatchData(BatchData newBatchData) { + batchDataList.add(newBatchData); + } + + public void appendBatchData(List newBatchDataList) { + batchDataList.addAll(newBatchDataList); + } + + public void setSignature(Signature newSignature) { + signature = newSignature; + } + + public void setTimestamp(Timestamp timestamp) { + this.timestamp = timestamp; + } + + @Override + public boolean equals(Object other) { + + if (!(other instanceof CompleteBatch)) { + return false; + } + + CompleteBatch otherBatch = (CompleteBatch) other; + + boolean result = true; + + if (beginBatchMessage == null) { + if (otherBatch.getBeginBatchMessage() != null) + return false; + } else { + result = result && beginBatchMessage.equals(otherBatch.getBeginBatchMessage()); + } + + if (batchDataList == null) { + if (otherBatch.getBatchDataList() != null) + return false; + } else { + result = result && batchDataList.equals(otherBatch.getBatchDataList()); + } + + if (signature == null) { + if (otherBatch.getSignature() != null) + return false; + } else { + result = result && signature.equals(otherBatch.getSignature()); + } + + if (timestamp == null) { + if (otherBatch.getTimestamp() != null) + return false; + } else { + result = result && timestamp.equals(otherBatch.getTimestamp()); + } + + return result; + + } + + @Override + public String toString() { + return "Batch " + beginBatchMessage.getSignerId().toString() + ":" + beginBatchMessage.getBatchId(); + } + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java new file mode 100644 index 0000000..4f25f59 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java @@ -0,0 +1,59 @@ +package meerkat.bulletinboard; + +import com.google.protobuf.Message; +import meerkat.crypto.Digest; +import meerkat.protobuf.BulletinBoardAPI.MessageID; +import meerkat.protobuf.BulletinBoardAPI.BatchData; + +import java.util.List; + + +/** + * Created by Arbel Deutsch Peled on 19-Dec-15. + * Wrapper class for digesting Batches in a standardized way + */ +public class GenericBatchDigest implements BatchDigest{ + + private Digest digest; + + public GenericBatchDigest(Digest digest) { + this.digest = digest; + } + + @Override + public void update(CompleteBatch completeBatch) { + + update(completeBatch.getBeginBatchMessage()); + + for (BatchData batchData : completeBatch.getBatchDataList()) { + update(batchData); + } + + } + + @Override + public byte[] digest() { + return digest.digest(); + } + + @Override + public MessageID digestAsMessageID() { + return digest.digestAsMessageID(); + } + + @Override + public void update(Message msg) { + digest.update(msg); + } + + @Override + public void reset() { + digest.reset(); + } + + @Override + public GenericBatchDigest clone() throws CloneNotSupportedException{ + return new GenericBatchDigest(digest.clone()); + } + +} diff --git a/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java new file mode 100644 index 0000000..7174b42 --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java @@ -0,0 +1,101 @@ +package meerkat.bulletinboard; + +import com.google.protobuf.ByteString; +import com.google.protobuf.Message; +import meerkat.crypto.DigitalSignature; +import meerkat.protobuf.BulletinBoardAPI.BatchData; +import meerkat.protobuf.Crypto; + +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidKeyException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.SignatureException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; + +/** + * Created by Arbel Deutsch Peled on 20-Dec-15. + * Wrapper class for signing and verifying Batch signatures in a standardized way + */ +public class GenericBatchDigitalSignature implements BatchDigitalSignature{ + + private DigitalSignature digitalSignature; + + public GenericBatchDigitalSignature(DigitalSignature digitalSignature) { + this.digitalSignature = digitalSignature; + } + + @Override + public void updateContent(CompleteBatch completeBatch) throws SignatureException { + + digitalSignature.updateContent(completeBatch.getBeginBatchMessage()); + for (BatchData batchData : completeBatch.getBatchDataList()) { + digitalSignature.updateContent(batchData); + } + + } + + @Override + public boolean verify(CompleteBatch completeBatch) throws SignatureException, CertificateException, InvalidKeyException { + + digitalSignature.initVerify(completeBatch.getSignature()); + updateContent(completeBatch); + return digitalSignature.verify(); + + } + + @Override + public void loadVerificationCertificates(InputStream certStream) throws CertificateException { + digitalSignature.loadVerificationCertificates(certStream); + } + + @Override + public void clearVerificationCertificates() { + digitalSignature.clearVerificationCertificates(); + } + + @Override + public void updateContent(Message msg) throws SignatureException { + digitalSignature.updateContent(msg); + } + + @Override + public Crypto.Signature sign() throws SignatureException { + return digitalSignature.sign(); + } + + @Override + public void initVerify(Crypto.Signature sig) throws CertificateException, InvalidKeyException { + digitalSignature.initVerify(sig); + } + + @Override + public boolean verify() { + return digitalSignature.verify(); + } + + @Override + public KeyStore.Builder getPKCS12KeyStoreBuilder(InputStream keyStream, char[] password) + throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException { + return digitalSignature.getPKCS12KeyStoreBuilder(keyStream, password); + } + + @Override + public void loadSigningCertificate(KeyStore.Builder keyStoreBuilder) throws IOException, CertificateException, UnrecoverableKeyException { + digitalSignature.loadSigningCertificate(keyStoreBuilder); + } + + @Override + public ByteString getSignerID() { + return digitalSignature.getSignerID(); + } + + @Override + public void clearSigningKey() { + digitalSignature.clearSigningKey(); + } + +} diff --git a/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java b/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java new file mode 100644 index 0000000..33deb3f --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java @@ -0,0 +1,64 @@ +package meerkat.comm; + +import com.google.protobuf.Message; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.util.LinkedList; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + * A input stream of Protobuf messages + */ +public class MessageInputStream{ + + private T.Builder builder; + + private InputStream in; + + MessageInputStream(InputStream in, Class type) throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException { + this.in = in; + this.builder = (T.Builder) type.getMethod("newBuilder").invoke(type); + } + + /** + * Factory class for actually creating a MessageInputStream + */ + public static class MessageInputStreamFactory { + + public static MessageInputStream createMessageInputStream(InputStream in, Class type) + throws IOException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { + + return new MessageInputStream<>(in, type); + + } + + } + + public T readMessage() throws IOException{ + + builder.clear(); + builder.mergeDelimitedFrom(in); + return (T) builder.build(); + + } + + public boolean isAvailable() throws IOException { + return (in.available() > 0); + } + + public List asList() throws IOException{ + + List list = new LinkedList<>(); + + while (isAvailable()){ + list.add(readMessage()); + } + + return list; + + } + +} diff --git a/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java b/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java new file mode 100644 index 0000000..b55bc8e --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java @@ -0,0 +1,24 @@ +package meerkat.comm; + +import com.google.protobuf.Message; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + * An output stream of Protobuf messages + */ +public class MessageOutputStream { + + private OutputStream out; + + public MessageOutputStream(OutputStream out) throws IOException { + this.out = out; + } + + public void writeMessage(T message) throws IOException { + message.writeDelimitedTo(out); + } + +} diff --git a/meerkat-common/src/main/java/meerkat/comm/Timestamp.java b/meerkat-common/src/main/java/meerkat/comm/Timestamp.java deleted file mode 100644 index 6c63854..0000000 --- a/meerkat-common/src/main/java/meerkat/comm/Timestamp.java +++ /dev/null @@ -1,7 +0,0 @@ -package meerkat.comm; - -/** - * Created by talm on 24/10/15. - */ -public class Timestamp { -} diff --git a/meerkat-common/src/main/java/meerkat/crypto/Digest.java b/meerkat-common/src/main/java/meerkat/crypto/Digest.java index 06b012c..b7d86dc 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/Digest.java +++ b/meerkat-common/src/main/java/meerkat/crypto/Digest.java @@ -1,6 +1,7 @@ package meerkat.crypto; import com.google.protobuf.Message; +import meerkat.protobuf.BulletinBoardAPI.MessageID; import java.security.MessageDigest; @@ -13,7 +14,13 @@ public interface Digest { * (copied from {@link MessageDigest#digest()}) * @return */ - byte[] digest(); + public byte[] digest(); + + /** + * Completes the hash computation and returns a MessageID Protobuf as output + * @return + */ + public MessageID digestAsMessageID(); /** * Updates the digest using the specified message (in serialized wire form) @@ -22,12 +29,12 @@ public interface Digest { * @param msg * @return */ - void update(Message msg); + public void update(Message msg); /** * Resets the digest for further use. */ - void reset(); + public void reset(); /** * Clone the current digest state diff --git a/meerkat-common/src/main/java/meerkat/crypto/DigitalSignature.java b/meerkat-common/src/main/java/meerkat/crypto/DigitalSignature.java index e7b64e5..76c32a6 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/DigitalSignature.java +++ b/meerkat-common/src/main/java/meerkat/crypto/DigitalSignature.java @@ -5,11 +5,13 @@ import com.google.protobuf.Message; import java.io.IOException; import java.io.InputStream; -import java.security.InvalidKeyException; import java.security.KeyStore; -import java.security.SignatureException; -import java.security.UnrecoverableKeyException; import java.security.cert.CertificateException; +import java.security.SignatureException; +import java.security.InvalidKeyException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; import static meerkat.protobuf.Crypto.*; /** @@ -71,6 +73,20 @@ public interface DigitalSignature { */ public boolean verify(); + /** + * Load a keystore from an input stream in PKCS12 format. + * + * @param keyStream + * @param password + * @return + * @throws IOException + * @throws CertificateException + * @throws KeyStoreException + * @throws NoSuchAlgorithmException + */ + public KeyStore.Builder getPKCS12KeyStoreBuilder(InputStream keyStream, char[] password) + throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException; + /** * Loads a private signing key. The keystore must include both the public and private * key parts. diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java index 887b8e8..ab8084b 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java @@ -30,7 +30,10 @@ import javax.security.auth.callback.UnsupportedCallbackException; * * This class is not thread-safe (each thread should have its own instance). */ -public class ECDSASignature extends GlobalCryptoSetup implements DigitalSignature { +public class ECDSASignature implements DigitalSignature { + + private static GlobalCryptoSetup globalCryptoSetup = GlobalCryptoSetup.getInstance(); + final Logger logger = LoggerFactory.getLogger(getClass()); final public static String KEYSTORE_TYPE = "PKCS12"; @@ -208,6 +211,7 @@ public class ECDSASignature extends GlobalCryptoSetup implements DigitalSignatur * @throws KeyStoreException * @throws NoSuchAlgorithmException */ + @Override public KeyStore.Builder getPKCS12KeyStoreBuilder(InputStream keyStream, char[] password) throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException { KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE); diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java index f9936c7..98d32bf 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java @@ -31,7 +31,10 @@ import java.util.Random; /** * Created by talm on 17/11/15. */ -public class ECElGamalEncryption extends GlobalCryptoSetup implements Encryption { +public class ECElGamalEncryption implements Encryption { + + private static GlobalCryptoSetup globalCryptoSetup = GlobalCryptoSetup.getInstance(); + final Logger logger = LoggerFactory.getLogger(getClass()); public final static String KEY_ALGORITHM = "ECDH"; diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/GlobalCryptoSetup.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/GlobalCryptoSetup.java index 4f2e7a5..3d12701 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/GlobalCryptoSetup.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/GlobalCryptoSetup.java @@ -10,13 +10,26 @@ import java.security.Security; /** * A class that performs required crypto setup */ -public class GlobalCryptoSetup { +public final class GlobalCryptoSetup { + + private static GlobalCryptoSetup globalCryptoSetup; + final static Logger logger = LoggerFactory.getLogger(GlobalCryptoSetup.class); - static boolean loadedBouncyCastle = false; - static Provider bouncyCastleProvider; + private boolean loadedBouncyCastle = false; + private Provider bouncyCastleProvider; - public static boolean hasSecp256k1Curve() { + private GlobalCryptoSetup() { doSetup(); } + + public static GlobalCryptoSetup getInstance() { + if (globalCryptoSetup == null) { + globalCryptoSetup = new GlobalCryptoSetup(); + } + + return globalCryptoSetup; + } + + public boolean hasSecp256k1Curve() { // For now we just check if the java version is at least 8 String[] version = System.getProperty("java.version").split("\\."); int major = Integer.parseInt(version[0]); @@ -24,9 +37,11 @@ public class GlobalCryptoSetup { return ((major > 1) || ((major > 0) && (minor > 7))); } - public static Provider getBouncyCastleProvider() { doSetup(); return bouncyCastleProvider; } + public Provider getBouncyCastleProvider() { + return bouncyCastleProvider; + } - public static synchronized void doSetup() { + public void doSetup() { if (bouncyCastleProvider == null) { bouncyCastleProvider = new BouncyCastleProvider(); // Make bouncycastle our default provider if we're running on a JVM version < 8 @@ -39,5 +54,4 @@ public class GlobalCryptoSetup { } } - public GlobalCryptoSetup() { doSetup(); } } diff --git a/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java b/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java index 4f60af3..a7723ec 100644 --- a/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java +++ b/meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java @@ -3,6 +3,7 @@ package meerkat.crypto.concrete; import com.google.protobuf.ByteString; import com.google.protobuf.Message; import meerkat.crypto.Digest; +import meerkat.protobuf.BulletinBoardAPI.MessageID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,7 +14,7 @@ import java.security.NoSuchAlgorithmException; /** * Created by talm on 11/9/15. */ -public class SHA256Digest extends GlobalCryptoSetup implements Digest { +public class SHA256Digest implements Digest { final Logger logger = LoggerFactory.getLogger(getClass()); public static final String SHA256 = "SHA-256"; @@ -60,6 +61,11 @@ public class SHA256Digest extends GlobalCryptoSetup implements Digest { return hash.digest(); } + @Override + public MessageID digestAsMessageID() { + return MessageID.newBuilder().setID(ByteString.copyFrom(digest())).build(); + } + @Override public void update(Message msg) { diff --git a/meerkat-common/src/main/java/meerkat/util/BulletinBoardMessageGenerator.java b/meerkat-common/src/main/java/meerkat/util/BulletinBoardMessageGenerator.java new file mode 100644 index 0000000..5ca3e0b --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/util/BulletinBoardMessageGenerator.java @@ -0,0 +1,98 @@ +package meerkat.util; + +import com.google.protobuf.ByteString; +import meerkat.crypto.DigitalSignature; +import meerkat.protobuf.BulletinBoardAPI.*; +import com.google.protobuf.Timestamp; + +import java.math.BigInteger; +import java.security.SignatureException; +import java.util.Arrays; +import java.util.Random; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + * This class contains methods used to generate random Bulletin Board Messages + */ +public class BulletinBoardMessageGenerator { + + private Random random; + + public BulletinBoardMessageGenerator(Random random) { + this.random = random; + } + + private byte randomByte(){ + return (byte) random.nextInt(); + } + + private String randomString(){ + return new BigInteger(130, random).toString(32); + } + + /** + * Generates a complete instance of a BulletinBoardMessage + * @param signers contains the (possibly multiple) credentials required to sign the message + * @param timestamp contains the time used in the message + * @param dataSize is the length of the data contained in the message + * @param tagNumber is the number of tags to generate + * @return a random, signed Bulletin Board Message containing random data and tags and the given timestamp + */ + + public BulletinBoardMessage generateRandomMessage(DigitalSignature[] signers, Timestamp timestamp, int dataSize, int tagNumber) + throws SignatureException { + + // Generate random data. + + byte[] data = new byte[dataSize]; + String[] tags = new String[tagNumber]; + + for (int i = 0; i < dataSize; i++) { + data[i] = randomByte(); + } + + for (int i = 0; i < tagNumber; i++) { + tags[i] = randomString(); + } + + UnsignedBulletinBoardMessage unsignedMessage = + UnsignedBulletinBoardMessage.newBuilder() + .setData(ByteString.copyFrom(data)) + .setTimestamp(timestamp) + .addAllTag(Arrays.asList(tags)) + .build(); + + BulletinBoardMessage.Builder messageBuilder = + BulletinBoardMessage.newBuilder() + .setMsg(unsignedMessage); + + for (int i = 0 ; i < signers.length ; i++) { + signers[i].updateContent(unsignedMessage); + messageBuilder.addSig(signers[i].sign()); + } + + return messageBuilder.build(); + + } + + /** + * Generates a complete instance of a BulletinBoardMessage + * @param signers contains the (possibly multiple) credentials required to sign the message + * @param dataSize is the length of the data contained in the message + * @param tagNumber is the number of tags to generate + * @return a random, signed Bulletin Board Message containing random data, tags and timestamp + */ + + public BulletinBoardMessage generateRandomMessage(DigitalSignature[] signers, int dataSize, int tagNumber) + throws SignatureException { + + Timestamp timestamp = Timestamp.newBuilder() + .setSeconds(random.nextLong()) + .setNanos(random.nextInt()) + .build(); + + return generateRandomMessage(signers, timestamp, dataSize, tagNumber); + + } + +} diff --git a/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java b/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java new file mode 100644 index 0000000..d8b362c --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java @@ -0,0 +1,107 @@ +package meerkat.util; + +import meerkat.protobuf.BulletinBoardAPI.*; + +import java.util.LinkedList; +import java.util.List; + +/** + * Created by Arbel Deutsch Peled on 16-Feb-16. + */ +public class BulletinBoardUtils { + + /** + * Searches the tags in the message for one that begins with given prefix + * @param message is the message to search + * @param prefix is the given prefix + * @return the tag without the prefix, if found, or null if not found + */ + public static String findTagWithPrefix(BulletinBoardMessage message, String prefix) { + + for (String tag : message.getMsg().getTagList()){ + if (tag.startsWith(prefix)) { + return tag.substring(prefix.length()); + } + } + + return null; + + } + + /** + * Searches the tags in a message for tags that do not contain a given list of prefixes + * @param message is the message to search + * @param prefixes is the list of prefixes + * @return a list of the tags that do *not* contain any of the given prefixes + */ + public static List removePrefixTags(BulletinBoardMessage message, Iterable prefixes) { + + if (prefixes == null) + return message.getMsg().getTagList(); + + List result = new LinkedList<>(); + + for (String tag : message.getMsg().getTagList()){ + + boolean found = false; + + for (String prefix : prefixes){ + if (tag.startsWith(prefix)){ + found = true; + break; + } + } + + if (!found) { + result.add(tag); + } + + } + + return result; + + } + + /** + * This method creates a Timestamp Protobuf from a time specification + * @param timeInMillis is the time to encode since the Epoch time in milliseconds + * @return a Timestamp Protobuf encoding of the given time + */ + public static com.google.protobuf.Timestamp toTimestampProto(long timeInMillis) { + + return com.google.protobuf.Timestamp.newBuilder() + .setSeconds(timeInMillis / 1000) + .setNanos((int) ((timeInMillis % 1000) * 1000000)) + .build(); + + } + + /** + * This method creates a Timestamp Protobuf from the current system time + * @return a Timestamp Protobuf encoding of the current system time + */ + public static com.google.protobuf.Timestamp toTimestampProto() { + + return toTimestampProto(System.currentTimeMillis()); + + } + + /** + * This method converts an SQL Timestamp object into a Protobuf Timestamp object + * @param sqlTimestamp is the SQL Timestamp + * @return an equivalent Protobuf Timestamp + */ + public static com.google.protobuf.Timestamp toTimestampProto(java.sql.Timestamp sqlTimestamp) { + return toTimestampProto(sqlTimestamp.getTime()); + } + + /** + * This method converts a Protobuf Timestamp object into an SQL Timestamp object + * @param protoTimestamp is the Protobuf Timestamp + * @return an equivalent SQL Timestamp + */ + public static java.sql.Timestamp toSQLTimestamp(com.google.protobuf.Timestamp protoTimestamp) { + return new java.sql.Timestamp(protoTimestamp.getSeconds() * 1000 + protoTimestamp.getNanos() / 1000000); + } + +} diff --git a/meerkat-common/src/main/java/meerkat/util/TimeStampComparator.java b/meerkat-common/src/main/java/meerkat/util/TimeStampComparator.java new file mode 100644 index 0000000..1dc207b --- /dev/null +++ b/meerkat-common/src/main/java/meerkat/util/TimeStampComparator.java @@ -0,0 +1,30 @@ +package meerkat.util; + +import com.google.protobuf.Timestamp; + +import java.util.Comparator; + +/** + * Created by Arbel Deutsch Peled on 20-Feb-16. + */ +public class TimestampComparator implements Comparator { + + @Override + public int compare(Timestamp o1, Timestamp o2) { + + if (o1.getSeconds() != o2.getSeconds()){ + + return o1.getSeconds() > o2.getSeconds() ? 2 : -2; + + } else if (o1.getNanos() != o2.getNanos()){ + + return o1.getNanos() > o2.getNanos() ? 1 : -1; + + } else{ + + return 0; + + } + + } +} diff --git a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto index 0fe35f8..fd95503 100644 --- a/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto +++ b/meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto @@ -5,6 +5,7 @@ package meerkat; option java_package = "meerkat.protobuf"; import 'meerkat/crypto.proto'; +import 'google/protobuf/timestamp.proto'; message BoolMsg { bool value = 1; @@ -21,11 +22,14 @@ message MessageID { } message UnsignedBulletinBoardMessage { - // Optional tags describing message + // Optional tags describing message; Used for message retrieval repeated string tag = 1; + // Timestamp of the message (as defined by client) + google.protobuf.Timestamp timestamp = 2; + // The actual content of the message - bytes data = 2; + bytes data = 3; } message BulletinBoardMessage { @@ -38,6 +42,7 @@ message BulletinBoardMessage { // Signature of message (and tags), excluding the entry number. repeated meerkat.Signature sig = 3; + } message BulletinBoardMessageList { @@ -50,13 +55,16 @@ enum FilterType { MSG_ID = 0; // Match exact message ID EXACT_ENTRY = 1; // Match exact entry number in database (chronological) MAX_ENTRY = 2; // Find all entries in database up to specified entry number (chronological) - SIGNER_ID = 3; // Find all entries in database that correspond to specific signature (signer) - TAG = 4; // Find all entries in database that have a specific tag + MIN_ENTRY = 3; // Find all entries in database starting from specified entry number (chronological) + SIGNER_ID = 4; // Find all entries in database that correspond to specific signature (signer) + TAG = 5; // Find all entries in database that have a specific tag + AFTER_TIME = 6; // Find all entries in database that occurred on or after a given timestamp + BEFORE_TIME = 7; // Find all entries in database that occurred on or before a given timestamp // NOTE: The MAX_MESSAGES filter must remain the last filter type // This is because the condition it specifies in an SQL statement must come last in the statement // Keeping it last here allows for easily sorting the filters and keeping the code general - MAX_MESSAGES = 5; // Return at most some specified number of messages + MAX_MESSAGES = 8; // Return at most some specified number of messages } message MessageFilter { @@ -68,6 +76,7 @@ message MessageFilter { int64 entry = 3; string tag = 4; int64 maxMessages = 5; + google.protobuf.Timestamp timestamp = 6; } } @@ -77,4 +86,74 @@ message MessageFilterList { // To be implemented using intersection ("AND") operations. repeated MessageFilter filter = 1; +} + +// This message is used to start a batch transfer to the Bulletin Board Server +message BeginBatchMessage { + bytes signerId = 1; // Unique signer identifier + int32 batchId = 2; // Unique identifier for the batch (unique per signer) + repeated string tag = 3; // Tags for the batch message +} + +// This message is used to finalize and sign a batch transfer to the Bulletin Board Server +message CloseBatchMessage { + int32 batchId = 1; // Unique identifier for the batch (unique per signer) + int32 batchLength = 2; // Number of messages in the batch + google.protobuf.Timestamp timestamp = 3; // Timestamp of the batch (as defined by client) + meerkat.Signature sig = 4; // Signature on the (ordered) batch messages +} + +// Container for single batch message data +message BatchData { + bytes data = 1; +} + +// List of BatchData; Only used for testing +message BatchDataList { + repeated BatchData data = 1; +} + +// These messages comprise a batch message +message BatchMessage { + bytes signerId = 1; // Unique signer identifier + int32 batchId = 2; // Unique identifier for the batch (unique per signer) + int32 serialNum = 3; // Location of the message in the batch: starting from 0 + BatchData data = 4; // Actual data +} + +// This message defines which batch to read and from which location to start reading +message BatchSpecificationMessage { + bytes signerId = 1; // Unique signer identifier + int32 batchId = 2; // Unique identifier for the batch (unique per signer) + int32 startPosition = 3; // Position in batch to start reading from +} + +// This message is used to define a single query to the server to ascertain whether or not the server is synched with the client +// up till a specified timestamp +message SingleSyncQuery { + + google.protobuf.Timestamp timeOfSync = 1; + int64 checksum = 2; + +} + +// This message defines a complete server sync query +message SyncQuery { + + MessageFilterList filterList = 1; + + repeated SingleSyncQuery query = 2; + +} + +// This message defines the server's response format to a sync query +message SyncQueryResponse { + + // Serial entry number of current last entry in database + // Set to zero (0) in case no query checksums match + int64 lastEntryNum = 1; + + // Largest value of timestamp for which the checksums match + google.protobuf.Timestamp lastTimeOfSync = 2; + } \ No newline at end of file diff --git a/meerkat-common/src/test/java/meerkat/comm/MessageStreamTest.java b/meerkat-common/src/test/java/meerkat/comm/MessageStreamTest.java new file mode 100644 index 0000000..58dc7c1 --- /dev/null +++ b/meerkat-common/src/test/java/meerkat/comm/MessageStreamTest.java @@ -0,0 +1,98 @@ +package meerkat.comm; + +import com.google.protobuf.*; +import meerkat.comm.MessageInputStream.MessageInputStreamFactory; +import meerkat.protobuf.BulletinBoardAPI.*; +import meerkat.protobuf.Crypto; +import meerkat.util.BulletinBoardMessageComparator; +import org.junit.Test; + +import java.io.*; +import java.lang.reflect.InvocationTargetException; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; + +/** + * Created by Arbel Deutsch Peled on 21-Feb-16. + * Tests for MessageInputStream and MessageOutputStream classes + */ +public class MessageStreamTest { + + @Test + public void testWithBulletinBoardMessages() { + + MessageOutputStream out; + MessageInputStream in; + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + BulletinBoardMessageComparator comparator = new BulletinBoardMessageComparator(); + + byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4}; + byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14}; + byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24}; + + try { + + out = new MessageOutputStream<>(stream); + + } catch (IOException e) { + + System.err.println(e.getMessage()); + assertThat("Error creating streams: " + e.getMessage(), false); + return; + + } + + + + BulletinBoardMessage message = BulletinBoardMessage.newBuilder() + .setEntryNum(1) + .setMsg(UnsignedBulletinBoardMessage.newBuilder() + .setData(ByteString.copyFrom(b1)) + .addTag("Test") + .addTag("1234") + .setTimestamp(com.google.protobuf.Timestamp.newBuilder() + .setSeconds(19823451) + .setNanos(2134) + .build()) + .build()) + .addSig(Crypto.Signature.newBuilder() + .setSignerId(ByteString.copyFrom(b2)) + .setData(ByteString.copyFrom(b3)) + .build()) + .build(); + + try { + + out.writeMessage(message); + + } catch (IOException e) { + + System.err.println(e.getMessage()); + assertThat("Error writing message: " + e.getMessage(), false); + + } + + try { + + in = MessageInputStreamFactory.createMessageInputStream( + new ByteArrayInputStream(stream.toByteArray()), + BulletinBoardMessage.class); + + assertThat("Retrieved message was not identical to send message", comparator.compare(message, in.readMessage()), is(equalTo(0))); + + } catch (IOException e) { + + System.err.println(e.getMessage()); + assertThat("Error reading message: " + e.getMessage(), false); + + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + System.err.println(e.getMessage()); + assertThat("Error creating input stream " + e.getMessage(), false); + } + + } + +} diff --git a/meerkat-common/src/test/java/meerkat/crypto/concrete/ECElGamalUtils.java b/meerkat-common/src/test/java/meerkat/crypto/concrete/ECElGamalUtils.java index 66e1647..e57c817 100644 --- a/meerkat-common/src/test/java/meerkat/crypto/concrete/ECElGamalUtils.java +++ b/meerkat-common/src/test/java/meerkat/crypto/concrete/ECElGamalUtils.java @@ -26,6 +26,9 @@ import java.security.spec.InvalidKeySpecException; * utilities for ECElgamal */ public class ECElGamalUtils { + + private static GlobalCryptoSetup globalCryptoSetup = GlobalCryptoSetup.getInstance(); + final static Logger logger = LoggerFactory.getLogger(ECElGamalUtils.class); public final static String ENCRYPTION_KEY_ALGORITHM = "ECDH"; @@ -43,7 +46,7 @@ public class ECElGamalUtils { try { KeyFactory fact = KeyFactory.getInstance(ENCRYPTION_KEY_ALGORITHM, - GlobalCryptoSetup.getBouncyCastleProvider()); + globalCryptoSetup.getBouncyCastleProvider()); PublicKey javaPk = fact.generatePublic(pubKeySpec); ConcreteCrypto.ElGamalPublicKey serializedPk = ConcreteCrypto.ElGamalPublicKey.newBuilder() .setSubjectPublicKeyInfo(ByteString.copyFrom(javaPk.getEncoded())).build(); diff --git a/restful-api-common/src/main/java/meerkat/rest/Constants.java b/restful-api-common/src/main/java/meerkat/rest/Constants.java index 73ed7d1..2c04248 100644 --- a/restful-api-common/src/main/java/meerkat/rest/Constants.java +++ b/restful-api-common/src/main/java/meerkat/rest/Constants.java @@ -5,8 +5,4 @@ package meerkat.rest; */ public interface Constants { public static final String MEDIATYPE_PROTOBUF = "application/x-protobuf"; - - public static final String BULLETIN_BOARD_SERVER_PATH = "/bbserver"; - public static final String READ_MESSAGES_PATH = "/readmessages"; - public static final String POST_MESSAGE_PATH = "/postmessage"; }