meerkat
/
meerkat-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mereging with master for getting the new timestamp uasge

Voter-Registry
Vladimir Eliezer Tokarev 2016-03-04 04:56:14 -08:00
parent 687c2c6d7e 50bcca8da3
commit 5c4b5a2150
78 changed files with 5508 additions and 988 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bulletin-board-client/src/main/java/meerkat/bulletinboard/BatchDataContainer.java Normal file
View File

@ -0,0 +1,25 @@
package meerkat.bulletinboard;
import com.google.protobuf.ByteString;
import meerkat.protobuf.BulletinBoardAPI.BatchData;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 17-Jan-16.
* Used to store the complete data required for sending a batch data list inside a single object
*/
public class BatchDataContainer {
public final byte[] signerId;
public final int batchId;
public final List<BatchData> batchDataList;
public final int startPosition;
public BatchDataContainer(byte[] signerId, int batchId, List<BatchData> batchDataList, int startPosition) {
this.signerId = signerId;
this.batchId = batchId;
this.batchDataList = batchDataList;
this.startPosition = startPosition;
}
}

82
bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJob.java
View File

@ -1,82 +0,0 @@
package meerkat.bulletinboard;
import com.google.protobuf.Message;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 09-Dec-15.
*
* This class specifies the job that is required of a Bulletin Board Client Worker
*/
public class BulletinClientJob {
public static enum JobType{
POST_MESSAGE, // Post a message to servers
READ_MESSAGES, // Read messages according to some given filter (any server will do)
GET_REDUNDANCY // Check the redundancy of a specific message in the databases
}
private List<String> serverAddresses;
private int minServers; // The minimal number of servers the job must be successful on for the job to be completed
private final JobType jobType;
private final Message payload; // The information associated with the job type
private int maxRetry; // Number of retries for this job; set to -1 for infinite retries
public BulletinClientJob(List<String> serverAddresses, int minServers, JobType jobType, Message payload, int maxRetry) {
this.serverAddresses = serverAddresses;
this.minServers = minServers;
this.jobType = jobType;
this.payload = payload;
this.maxRetry = maxRetry;
}
public void updateServerAddresses(List<String> newServerAdresses) {
this.serverAddresses = newServerAdresses;
}
public List<String> getServerAddresses() {
return serverAddresses;
}
public int getMinServers() {
return minServers;
}
public JobType getJobType() {
return jobType;
}
public Message getPayload() {
return payload;
}
public int getMaxRetry() {
return maxRetry;
}
public void shuffleAddresses() {
Collections.shuffle(serverAddresses);
}
public void decMinServers(){
minServers--;
}
public void decMaxRetry(){
if (maxRetry > 0) {
maxRetry--;
}
}
public boolean isRetry(){
return (maxRetry != 0);
}
}

29
bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJobResult.java
View File

@ -1,29 +0,0 @@
package meerkat.bulletinboard;
import com.google.protobuf.Message;
/**
* Created by Arbel Deutsch Peled on 09-Dec-15.
*
* This class contains the end status and result of a Bulletin Board Client Job.
*/
public final class BulletinClientJobResult {
private final BulletinClientJob job; // Stores the job the result refers to
private final Message result; // The result of the job; valid only if success==true
public BulletinClientJobResult(BulletinClientJob job, Message result) {
this.job = job;
this.result = result;
}
public BulletinClientJob getJob() {
return job;
}
public Message getResult() {
return result;
}
}

223
bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java
View File

@ -1,217 +1,38 @@
package meerkat.bulletinboard;
import com.google.protobuf.Message;
import meerkat.comm.CommunicationException;
import meerkat.crypto.Digest;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.rest.Constants;
import meerkat.rest.ProtobufMessageBodyReader;
import meerkat.rest.ProtobufMessageBodyWriter;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Callable;
/**
* Created by Arbel Deutsch Peled on 09-Dec-15.
*
* This class implements the actual communication with the Bulletin Board Servers.
* This class handles bulletin client work.
* It is meant to be used in a multi-threaded environment.
*/
//TODO: Maybe make this abstract and inherit from it.
public class BulletinClientWorker implements Callable<BulletinClientJobResult> {
public abstract class BulletinClientWorker<IN> {
private final BulletinClientJob job; // The requested job to be handled
protected final IN payload; // Payload of the job
public BulletinClientWorker(BulletinClientJob job){
this.job = job;
private int maxRetry; // Number of retries for this job; set to -1 for infinite retries
public BulletinClientWorker(IN payload, int maxRetry) {
this.payload = payload;
this.maxRetry = maxRetry;
}
// This resource enabled creation of a single Client per thread.
private static final ThreadLocal<Client> clientLocal =
new ThreadLocal<Client> () {
@Override protected Client initialValue() {
Client client;
client = ClientBuilder.newClient();
client.register(ProtobufMessageBodyReader.class);
client.register(ProtobufMessageBodyWriter.class);
public IN getPayload() {
return payload;
}
return client;
}
};
// This resource enables creation of a single Digest per thread.
private static final ThreadLocal<Digest> digestLocal =
new ThreadLocal<Digest> () {
@Override protected Digest initialValue() {
Digest digest;
digest = new SHA256Digest(); //TODO: Make this generic.
return digest;
}
};
/**
* This method carries out the actual communication with the servers via HTTP Post
* It accesses the servers according to the job it received and updates said job as it goes
* The method will only iterate once through the server list, removing servers from the list when they are no longer required
* In a POST_MESSAGE job: successful post to a server results in removing the server from the list
* In a GET_REDUNDANCY job: no server is removed from the list and the (absolute) number of servers in which the message was found is returned
* In a READ_MESSAGES job: successful retrieval from any server terminates the method and returns the received values; The list is not changed
* @return The original job, modified to fit the current state and the required output (if any) of the operation
* @throws IllegalArgumentException
* @throws CommunicationException
*/
public BulletinClientJobResult call() throws IllegalArgumentException, CommunicationException{
Client client = clientLocal.get();
Digest digest = digestLocal.get();
WebTarget webTarget;
Response response;
String requestPath;
Message msg;
List<String> serverAddresses = new LinkedList<String>(job.getServerAddresses());
Message payload = job.getPayload();
BulletinBoardMessageList msgList;
int count = 0; // Used to count number of servers which contain the required message in a GET_REDUNDANCY request.
job.shuffleAddresses(); // This is done to randomize the order of access to servers primarily for READ operations
// Prepare the request.
switch(job.getJobType()) {
case POST_MESSAGE:
// Make sure the payload is a BulletinBoardMessage
if (!(payload instanceof BulletinBoardMessage)) {
throw new IllegalArgumentException("Cannot post an object that is not an instance of BulletinBoardMessage");
}
msg = payload;
requestPath = Constants.POST_MESSAGE_PATH;
break;
case READ_MESSAGES:
// Make sure the payload is a MessageFilterList
if (!(payload instanceof MessageFilterList)) {
throw new IllegalArgumentException("Read failed: an instance of MessageFilterList is required as payload for a READ_MESSAGES operation");
}
msg = payload;
requestPath = Constants.READ_MESSAGES_PATH;
break;
case GET_REDUNDANCY:
// Make sure the payload is a MessageId
if (!(payload instanceof MessageID)) {
throw new IllegalArgumentException("Cannot search for an object that is not an instance of MessageID");
}
requestPath = Constants.READ_MESSAGES_PATH;
msg = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(((MessageID) payload).getID())
.build()
).build();
break;
default:
throw new IllegalArgumentException("Unsupported job type");
}
// Iterate through servers
Iterator<String> addressIterator = serverAddresses.iterator();
while (addressIterator.hasNext()) {
// Send request to Server
String address = addressIterator.next();
webTarget = client.target(address).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(requestPath);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF));
// Retrieve answer
switch(job.getJobType()) {
case POST_MESSAGE:
try {
response.readEntity(BoolMsg.class); // If a BoolMsg entity is returned: the post was successful
addressIterator.remove(); // Post to this server succeeded: remove server from list
job.decMinServers();
} catch (ProcessingException | IllegalStateException e) {} // Post to this server failed: retry next time
finally {
response.close();
}
break;
case GET_REDUNDANCY:
try {
msgList = response.readEntity(BulletinBoardMessageList.class); // If a BulletinBoardMessageList is returned: the read was successful
if (msgList.getMessageList().size() > 0){ // Message was found in the server.
count++;
}
} catch (ProcessingException | IllegalStateException e) {} // Read failed: try with next server
finally {
response.close();
}
break;
case READ_MESSAGES:
try {
msgList = response.readEntity(BulletinBoardMessageList.class); // If a BulletinBoardMessageList is returned: the read was successful
return new BulletinClientJobResult(job, msgList); // Return the result
} catch (ProcessingException | IllegalStateException e) {} // Read failed: try with next server
finally {
response.close();
}
break;
}
}
// Return result (if haven't done so yet)
switch(job.getJobType()) {
case POST_MESSAGE:
// The job now contains the information required to ascertain whether enough server posts have succeeded
// It will also contain the list of servers in which the post was not successful
job.updateServerAddresses(serverAddresses);
return new BulletinClientJobResult(job, null);
case GET_REDUNDANCY:
// Return the number of servers in which the message was found
// The job now contains the list of these servers
return new BulletinClientJobResult(job, IntMsg.newBuilder().setValue(count).build());
case READ_MESSAGES:
// A successful operation would have already returned an output
// Therefore: no server access was successful
throw new CommunicationException("Could not access any server");
default: // This is required for successful compilation
throw new IllegalArgumentException("Unsupported job type");
public int getMaxRetry() {
return maxRetry;
}
public void decMaxRetry(){
if (maxRetry > 0) {
maxRetry--;
}
}
public boolean isRetry(){
return (maxRetry != 0);
}
}

104
bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java Normal file
View File

@ -0,0 +1,104 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.FutureCallback;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by Arbel Deutsch Peled on 09-Dec-15.
*
* This is a general class for handling multi-server work
* It utilizes Single Server Clients to perform the actual per-server work
*/
public abstract class MultiServerWorker<IN, OUT> extends BulletinClientWorker<IN> implements Runnable, FutureCallback<OUT>{
private final List<SingleServerBulletinBoardClient> clients;
protected AtomicInteger minServers; // The minimal number of servers the job must be successful on for the job to be completed
protected AtomicInteger maxFailedServers; // The maximal number of allowed server failures
private AtomicBoolean returnedResult;
private final FutureCallback<OUT> futureCallback;
/**
* Constructor
* @param clients contains a list of Single Server clients to handle requests
* @param shuffleClients is a boolean stating whether or not it is needed to shuffle the clients
* @param minServers is the minimal amount of servers needed in order to successfully complete the job
* @param payload is the payload for the job
* @param maxRetry is the maximal per-server retry count
* @param futureCallback contains the callback methods used to report the result back to the client
*/
public MultiServerWorker(List<SingleServerBulletinBoardClient> clients, boolean shuffleClients,
int minServers, IN payload, int maxRetry,
FutureCallback<OUT> futureCallback) {
super(payload,maxRetry);
this.clients = clients;
if (shuffleClients){
Collections.shuffle(clients);
}
this.minServers = new AtomicInteger(minServers);
maxFailedServers = new AtomicInteger(clients.size() - minServers);
this.futureCallback = futureCallback;
returnedResult = new AtomicBoolean(false);
}
/**
* Constructor overload without client shuffling
*/
public MultiServerWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, IN payload, int maxRetry,
FutureCallback<OUT> futureCallback) {
this(clients, false, minServers, payload, maxRetry, futureCallback);
}
/**
* Used to report a successful operation to the client
* Only reports once to the client
* @param result is the result
*/
protected void succeed(OUT result){
if (returnedResult.compareAndSet(false, true)) {
futureCallback.onSuccess(result);
}
}
/**
* Used to report a failed operation to the client
* Only reports once to the client
* @param t contains the error/exception that occurred
*/
protected void fail(Throwable t){
if (returnedResult.compareAndSet(false, true)) {
futureCallback.onFailure(t);
}
}
/**
* Used by implementations to get a Single Server Client iterator
* @return the requested iterator
*/
protected Iterator<SingleServerBulletinBoardClient> getClientIterator() {
return clients.iterator();
}
protected int getClientNumber() {
return clients.size();
}
}

42
bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java
View File

@ -5,8 +5,7 @@ import meerkat.comm.CommunicationException;
import meerkat.crypto.Digest;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Voting;
import meerkat.protobuf.Voting.BulletinBoardClientParams;
import meerkat.protobuf.Voting.*;
import meerkat.rest.*;
import java.util.List;
@ -17,25 +16,28 @@ import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import static meerkat.bulletinboard.BulletinBoardConstants.*;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
* Implements BulletinBoardClient interface in a simple, straightforward manner
*/
public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
public class SimpleBulletinBoardClient implements BulletinBoardClient{
private List<String> meerkatDBs;
protected List<String> meerkatDBs;
private Client client;
protected Client client;
private Digest digest;
protected Digest digest;
/**
* Stores database locations and initializes the web Client
* @param clientParams contains the data needed to access the DBs
*/
// @Override
public void init(Voting.BulletinBoardClientParams clientParams) {
@Override
public void init(BulletinBoardClientParams clientParams) {
meerkatDBs = clientParams.getBulletinBoardAddressList();
this.meerkatDBs = clientParams.getBulletinBoardAddressList();
client = ClientBuilder.newClient();
client.register(ProtobufMessageBodyReader.class);
@ -52,7 +54,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
* @return the message ID for later retrieval
* @throws CommunicationException
*/
// @Override
@Override
public MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException {
WebTarget webTarget;
@ -61,7 +63,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
// Post message to all databases
try {
for (String db : meerkatDBs) {
webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.POST_MESSAGE_PATH);
webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(POST_MESSAGE_PATH);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF));
// Only consider valid responses
@ -88,7 +90,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
* @param id is the requested message ID
* @return the number of DBs in which retrieval was successful
*/
// @Override
@Override
public float getRedundancy(MessageID id) {
WebTarget webTarget;
Response response;
@ -104,7 +106,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
for (String db : meerkatDBs) {
try {
webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH);
webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF));
@ -123,9 +125,9 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
* If at the operation is successful for some DB: return the results and stop iterating
* If no operation is successful: return null (NOT blank list)
* @param filterList return only messages that match the filters (null means no filtering).
* @return
* @return the list of Bulletin Board messages that are returned from a server
*/
// @Override
@Override
public List<BulletinBoardMessage> readMessages(MessageFilterList filterList) {
WebTarget webTarget;
Response response;
@ -138,7 +140,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
for (String db : meerkatDBs) {
try {
webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH);
webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF));
@ -154,8 +156,8 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
return null;
}
// @Override
// public void registerNewMessageCallback(MessageCallback callback, MessageFilterList filterList) {
// callback.handleNewMessage(readMessages(filterList));
// }
public void close() {
client.close();
}
}

605
bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java Normal file
View File

@ -0,0 +1,605 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.ByteString;
import meerkat.bulletinboard.workers.singleserver.*;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Voting.BulletinBoardClientParams;
import meerkat.util.BulletinBoardUtils;
import javax.ws.rs.NotFoundException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by Arbel Deutsch Peled on 28-Dec-15.
*
* This class implements the asynchronous Bulletin Board Client interface
* It only handles a single Bulletin Board Server
* If the list of servers contains more than one server: the server actually used is the first one
* The class further implements a delayed access to the server after a communication error occurs
*/
public class SingleServerBulletinBoardClient extends SimpleBulletinBoardClient implements AsyncBulletinBoardClient {
private final int MAX_RETRIES = 11;
private ListeningScheduledExecutorService executorService;
protected BatchDigest batchDigest;
private long lastServerErrorTime;
private final long failDelayInMilliseconds;
private final long subscriptionIntervalInMilliseconds;
/**
* Notify the client that a job has failed
* This makes new scheduled jobs be scheduled for a later time (after the given delay)
*/
protected void fail() {
// Update last fail time
lastServerErrorTime = System.currentTimeMillis();
}
/**
* This method adds a worker to the scheduled queue of the threadpool
* If the server is in an accessible state: the job is submitted for immediate handling
* If the server is not accessible: the job is scheduled for a later time
* @param worker is the worker that should be scheduled for work
* @param callback is the class containing callbacks for handling job completion/failure
*/
protected void scheduleWorker(SingleServerWorker worker, FutureCallback callback){
long timeSinceLastServerError = System.currentTimeMillis() - lastServerErrorTime;
if (timeSinceLastServerError >= failDelayInMilliseconds) {
// Schedule for immediate processing
Futures.addCallback(executorService.submit(worker), callback);
} else {
// Schedule for processing immediately following delay expiry
Futures.addCallback(executorService.schedule(
worker,
failDelayInMilliseconds - timeSinceLastServerError,
TimeUnit.MILLISECONDS),
callback);
}
}
/**
* Inner class for handling simple operation results and retrying if needed
*/
class RetryCallback<T> implements FutureCallback<T> {
private final SingleServerWorker worker;
private final FutureCallback<T> futureCallback;
public RetryCallback(SingleServerWorker worker, FutureCallback<T> futureCallback) {
this.worker = worker;
this.futureCallback = futureCallback;
}
@Override
public void onSuccess(T result) {
futureCallback.onSuccess(result);
}
@Override
public void onFailure(Throwable t) {
// Notify client about failure
fail();
// Check if another attempt should be made
worker.decMaxRetry();
if (worker.isRetry()) {
// Perform another attempt
scheduleWorker(worker, this);
} else {
// No more retries: notify caller about failure
futureCallback.onFailure(t);
}
}
}
/**
* This callback ties together all the per-batch-data callbacks into a single callback
* It reports success back to the user only if all of the batch-data were successfully posted
* If any batch-data fails to post: this callback reports failure
*/
class PostBatchDataListCallback implements FutureCallback<Boolean> {
private final FutureCallback<Boolean> callback;
private AtomicInteger batchDataRemaining;
private AtomicBoolean aggregatedResult;
public PostBatchDataListCallback(int batchDataLength, FutureCallback<Boolean> callback) {
this.callback = callback;
this.batchDataRemaining = new AtomicInteger(batchDataLength);
this.aggregatedResult = new AtomicBoolean(false);
}
@Override
public void onSuccess(Boolean result) {
if (result){
this.aggregatedResult.set(true);
}
if (batchDataRemaining.decrementAndGet() == 0){
callback.onSuccess(this.aggregatedResult.get());
}
}
@Override
public void onFailure(Throwable t) {
// Notify caller about failure
callback.onFailure(t);
}
}
/**
* This callback ties together the different parts of a CompleteBatch as they arrive from the server
* It assembles a CompleteBatch from the parts and sends it to the user if all parts arrived
* If any part fails to arrive: it invokes the onFailure method
*/
class CompleteBatchReadCallback {
private final FutureCallback<CompleteBatch> callback;
private List<BatchData> batchDataList;
private BulletinBoardMessage batchMessage;
private AtomicInteger remainingQueries;
private AtomicBoolean failed;
public CompleteBatchReadCallback(FutureCallback<CompleteBatch> callback) {
this.callback = callback;
remainingQueries = new AtomicInteger(2);
failed = new AtomicBoolean(false);
}
protected void combineAndReturn() {
final String[] prefixes = {
BulletinBoardConstants.BATCH_ID_TAG_PREFIX,
BulletinBoardConstants.BATCH_TAG};
if (remainingQueries.decrementAndGet() == 0){
String batchIdStr = BulletinBoardUtils.findTagWithPrefix(batchMessage, BulletinBoardConstants.BATCH_ID_TAG_PREFIX);
if (batchIdStr == null){
callback.onFailure(new CommunicationException("Server returned invalid message with no Batch ID tag"));
}
BeginBatchMessage beginBatchMessage =
BeginBatchMessage.newBuilder()
.setSignerId(batchMessage.getSig(0).getSignerId())
.setBatchId(Integer.parseInt(batchIdStr))
.addAllTag(BulletinBoardUtils.removePrefixTags(batchMessage, Arrays.asList(prefixes)))
.build();
callback.onSuccess(new CompleteBatch(beginBatchMessage, batchDataList, batchMessage.getSig(0)));
}
}
protected void fail(Throwable t) {
if (failed.compareAndSet(false, true)) {
callback.onFailure(t);
}
}
/**
* @return a FutureCallback for the Batch Data List that ties to this object
*/
public FutureCallback<List<BatchData>> asBatchDataListFutureCallback() {
return new FutureCallback<List<BatchData>>() {
@Override
public void onSuccess(List<BatchData> result) {
batchDataList = result;
combineAndReturn();
}
@Override
public void onFailure(Throwable t) {
fail(t);
}
};
}
/**
* @return a FutureCallback for the Bulletin Board Message that ties to this object
*/
public FutureCallback<List<BulletinBoardMessage>> asBulletinBoardMessageListFutureCallback() {
return new FutureCallback<List<BulletinBoardMessage>>() {
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
if (result.size() < 1){
onFailure(new IllegalArgumentException("Server returned empty message list"));
return;
}
batchMessage = result.get(0);
combineAndReturn();
}
@Override
public void onFailure(Throwable t) {
fail(t);
}
};
}
}
/**
* Inner class for handling returned values of subscription operations
* This class's methods also ensure continued operation of the subscription
*/
class SubscriptionCallback implements FutureCallback<List<BulletinBoardMessage>> {
private SingleServerReadMessagesWorker worker;
private final MessageHandler messageHandler;
private MessageFilterList.Builder filterBuilder;
public SubscriptionCallback(SingleServerReadMessagesWorker worker, MessageHandler messageHandler) {
this.worker = worker;
this.messageHandler = messageHandler;
filterBuilder = worker.getPayload().toBuilder();
}
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
// Report new messages to user
messageHandler.handleNewMessages(result);
// Remove last filter from list (MIN_ENTRY one)
filterBuilder.removeFilter(filterBuilder.getFilterCount() - 1);
// Add updated MIN_ENTRY filter (entry number is successor of last received entry's number)
filterBuilder.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MIN_ENTRY)
.setEntry(result.get(result.size() - 1).getEntryNum() + 1)
.build());
// Create new worker with updated task
worker = new SingleServerReadMessagesWorker(worker.serverAddress, filterBuilder.build(), 1);
// Schedule the worker
scheduleWorker(worker, this);
}
@Override
public void onFailure(Throwable t) {
// Notify client about failure
fail();
// Reschedule exact same task
scheduleWorker(worker, this);
}
}
public SingleServerBulletinBoardClient(int threadPoolSize, long failDelayInMilliseconds, long subscriptionIntervalInMilliseconds) {
executorService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(threadPoolSize));
this.failDelayInMilliseconds = failDelayInMilliseconds;
this.subscriptionIntervalInMilliseconds = subscriptionIntervalInMilliseconds;
// Set server error time to a time sufficiently in the past to make new jobs go through
lastServerErrorTime = System.currentTimeMillis() - failDelayInMilliseconds;
}
/**
* Stores database location, initializes the web Client and
* @param clientParams contains the data needed to access the DBs
*/
@Override
public void init(BulletinBoardClientParams clientParams) {
// Perform usual setup
super.init(clientParams);
// Wrap the Digest into a BatchDigest
batchDigest = new GenericBatchDigest(digest);
// Remove all but first DB address
String dbAddress = meerkatDBs.get(0);
meerkatDBs = new LinkedList<>();
meerkatDBs.add(dbAddress);
}
@Override
public MessageID postMessage(BulletinBoardMessage msg, FutureCallback<Boolean> callback) {
// Create worker with redundancy 1 and MAX_RETRIES retries
SingleServerPostMessageWorker worker = new SingleServerPostMessageWorker(meerkatDBs.get(0), msg, MAX_RETRIES);
// Submit worker and create callback
scheduleWorker(worker, new RetryCallback<>(worker, callback));
// Calculate the correct message ID and return it
batchDigest.reset();
batchDigest.update(msg.getMsg());
return batchDigest.digestAsMessageID();
}
private class PostBatchDataCallback implements FutureCallback<Boolean> {
private final CompleteBatch completeBatch;
private final FutureCallback<Boolean> callback;
public PostBatchDataCallback(CompleteBatch completeBatch, FutureCallback<Boolean> callback) {
this.completeBatch = completeBatch;
this.callback = callback;
}
@Override
public void onSuccess(Boolean msg) {
closeBatch(
completeBatch.getCloseBatchMessage(),
callback
);
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
}
private class BeginBatchCallback implements FutureCallback<Boolean> {
private final CompleteBatch completeBatch;
private final FutureCallback<Boolean> callback;
public BeginBatchCallback(CompleteBatch completeBatch, FutureCallback<Boolean> callback) {
this.completeBatch = completeBatch;
this.callback = callback;
}
@Override
public void onSuccess(Boolean msg) {
postBatchData(
completeBatch.getBeginBatchMessage().getSignerId(),
completeBatch.getBeginBatchMessage().getBatchId(),
completeBatch.getBatchDataList(),
0,
new PostBatchDataCallback(completeBatch,callback));
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
}
@Override
public MessageID postBatch(CompleteBatch completeBatch, FutureCallback<Boolean> callback) {
beginBatch(
completeBatch.getBeginBatchMessage(),
new BeginBatchCallback(completeBatch, callback)
);
batchDigest.update(completeBatch);
return batchDigest.digestAsMessageID();
}
@Override
public void beginBatch(BeginBatchMessage beginBatchMessage, FutureCallback<Boolean> callback) {
// Create worker with redundancy 1 and MAX_RETRIES retries
SingleServerBeginBatchWorker worker =
new SingleServerBeginBatchWorker(meerkatDBs.get(0), beginBatchMessage, MAX_RETRIES);
// Submit worker and create callback
scheduleWorker(worker, new RetryCallback<>(worker, callback));
}
@Override
public void postBatchData(ByteString signerId, int batchId, List<BatchData> batchDataList,
int startPosition, FutureCallback<Boolean> callback) {
BatchMessage.Builder builder = BatchMessage.newBuilder()
.setSignerId(signerId)
.setBatchId(batchId);
// Create a unified callback to aggregate successful posts
PostBatchDataListCallback listCallback = new PostBatchDataListCallback(batchDataList.size(), callback);
// Iterate through data list
for (BatchData data : batchDataList) {
builder.setSerialNum(startPosition).setData(data);
// Create worker with redundancy 1 and MAX_RETRIES retries
SingleServerPostBatchWorker worker =
new SingleServerPostBatchWorker(meerkatDBs.get(0), builder.build(), MAX_RETRIES);
// Create worker with redundancy 1 and MAX_RETRIES retries
scheduleWorker(worker, new RetryCallback<>(worker, listCallback));
// Increment position in batch
startPosition++;
}
}
@Override
public void postBatchData(ByteString signerId, int batchId, List<BatchData> batchDataList, FutureCallback<Boolean> callback) {
postBatchData(signerId, batchId, batchDataList, 0, callback);
}
@Override
public void postBatchData(byte[] signerId, int batchId, List<BatchData> batchDataList,
int startPosition, FutureCallback<Boolean> callback) {
postBatchData(ByteString.copyFrom(signerId), batchId, batchDataList, startPosition, callback);
}
@Override
public void postBatchData(byte[] signerId, int batchId, List<BatchData> batchDataList, FutureCallback<Boolean> callback) {
postBatchData(signerId, batchId, batchDataList, 0, callback);
}
@Override
public void closeBatch(CloseBatchMessage closeBatchMessage, FutureCallback<Boolean> callback) {
// Create worker with redundancy 1 and MAX_RETRIES retries
SingleServerCloseBatchWorker worker =
new SingleServerCloseBatchWorker(meerkatDBs.get(0), closeBatchMessage, MAX_RETRIES);
// Submit worker and create callback
scheduleWorker(worker, new RetryCallback<>(worker, callback));
}
@Override
public void getRedundancy(MessageID id, FutureCallback<Float> callback) {
// Create worker with no retries
SingleServerGetRedundancyWorker worker = new SingleServerGetRedundancyWorker(meerkatDBs.get(0), id, 1);
// Submit job and create callback
scheduleWorker(worker, new RetryCallback<>(worker, callback));
}
@Override
public void readMessages(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
// Create job with no retries
SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterList, 1);
// Submit job and create callback
scheduleWorker(worker, new RetryCallback<>(worker, callback));
}
@Override
public void readBatch(BatchSpecificationMessage batchSpecificationMessage, FutureCallback<CompleteBatch> callback) {
// Create job with no retries for retrieval of the Bulletin Board Message that defines the batch
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag(BulletinBoardConstants.BATCH_TAG)
.build())
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag(BulletinBoardConstants.BATCH_ID_TAG_PREFIX + batchSpecificationMessage.getBatchId())
.build())
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.SIGNER_ID)
.setId(batchSpecificationMessage.getSignerId())
.build())
.build();
SingleServerReadMessagesWorker messageWorker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterList, 1);
// Create job with no retries for retrieval of the Batch Data List
SingleServerReadBatchWorker batchWorker = new SingleServerReadBatchWorker(meerkatDBs.get(0), batchSpecificationMessage, 1);
// Create callback that will combine the two worker products
CompleteBatchReadCallback completeBatchReadCallback = new CompleteBatchReadCallback(callback);
// Submit jobs with wrapped callbacks
scheduleWorker(messageWorker, new RetryCallback<>(messageWorker, completeBatchReadCallback.asBulletinBoardMessageListFutureCallback()));
scheduleWorker(batchWorker, new RetryCallback<>(batchWorker, completeBatchReadCallback.asBatchDataListFutureCallback()));
}
@Override
public void subscribe(MessageFilterList filterList, MessageHandler messageHandler) {
// Remove all existing MIN_ENTRY filters and create new one that starts at 0
MessageFilterList.Builder filterListBuilder = filterList.toBuilder();
Iterator<MessageFilter> iterator = filterListBuilder.getFilterList().iterator();
while (iterator.hasNext()) {
MessageFilter filter = iterator.next();
if (filter.getType() == FilterType.MIN_ENTRY){
iterator.remove();
}
}
filterListBuilder.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MIN_ENTRY)
.setEntry(0)
.build());
// Create job with no retries
SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(meerkatDBs.get(0), filterListBuilder.build(), 1);
// Submit job and create callback
scheduleWorker(worker, new SubscriptionCallback(worker, messageHandler));
}
@Override
public void close() {
super.close();
executorService.shutdown();
}
}

39
bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java Normal file
View File

@ -0,0 +1,39 @@
package meerkat.bulletinboard;
import meerkat.rest.ProtobufMessageBodyReader;
import meerkat.rest.ProtobufMessageBodyWriter;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import java.util.concurrent.Callable;
/**
* Created by Arbel Deutsch Peled on 02-Jan-16.
*/
public abstract class SingleServerWorker<IN, OUT> extends BulletinClientWorker<IN> implements Callable<OUT>{
// This resource enabled creation of a single Client per thread.
protected static final ThreadLocal<Client> clientLocal =
new ThreadLocal<Client> () {
@Override protected Client initialValue() {
Client client;
client = ClientBuilder.newClient();
client.register(ProtobufMessageBodyReader.class);
client.register(ProtobufMessageBodyWriter.class);
return client;
}
};
protected final String serverAddress;
public SingleServerWorker(String serverAddress, IN payload, int maxRetry) {
super(payload, maxRetry);
this.serverAddress = serverAddress;
}
public String getServerAddress() {
return serverAddress;
}
}

211
bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java
View File

@ -1,39 +1,44 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.*;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.ByteString;
import meerkat.bulletinboard.callbacks.GetRedundancyFutureCallback;
import meerkat.bulletinboard.callbacks.PostMessageFutureCallback;
import meerkat.bulletinboard.callbacks.ReadMessagesFutureCallback;
import meerkat.comm.CommunicationException;
import meerkat.crypto.Digest;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Voting;
import meerkat.bulletinboard.workers.multiserver.*;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Voting.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
* Thread-based implementation of a Bulletin Board Client.
* Thread-based implementation of a Async Bulletin Board Client.
* Features:
* 1. Handles tasks concurrently.
* 2. Retries submitting
*/
public class ThreadedBulletinBoardClient implements BulletinBoardClient {
public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient implements AsyncBulletinBoardClient {
private final static int THREAD_NUM = 10;
ListeningExecutorService listeningExecutor;
// Executor service for handling jobs
private final static int JOBS_THREAD_NUM = 5;
private ExecutorService executorService;
private Digest digest;
// Per-server clients
private List<SingleServerBulletinBoardClient> clients;
private List<String> meerkatDBs;
private String postSubAddress;
private String readSubAddress;
private BatchDigest batchDigest;
private final static int POST_MESSAGE_RETRY_NUM = 3;
private final static int READ_MESSAGES_RETRY_NUM = 1;
private final static int GET_REDUNDANCY_RETRY_NUM = 1;
private static final int SERVER_THREADPOOL_SIZE = 5;
private static final long FAIL_DELAY = 5000;
private static final long SUBSCRIPTION_INTERVAL = 10000;
private int minAbsoluteRedundancy;
@ -44,15 +49,29 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient {
* @param clientParams contains the required information
*/
@Override
public void init(Voting.BulletinBoardClientParams clientParams) {
public void init(BulletinBoardClientParams clientParams) {
meerkatDBs = clientParams.getBulletinBoardAddressList();
super.init(clientParams);
minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * meerkatDBs.size());
batchDigest = new GenericBatchDigest(digest);
listeningExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(THREAD_NUM));
minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * clientParams.getBulletinBoardAddressCount());
digest = new SHA256Digest();
executorService = Executors.newFixedThreadPool(JOBS_THREAD_NUM);
clients = new ArrayList<>(clientParams.getBulletinBoardAddressCount());
for (String address : clientParams.getBulletinBoardAddressList()){
SingleServerBulletinBoardClient client =
new SingleServerBulletinBoardClient(SERVER_THREADPOOL_SIZE, FAIL_DELAY, SUBSCRIPTION_INTERVAL);
client.init(BulletinBoardClientParams.newBuilder()
.addBulletinBoardAddress(address)
.build());
clients.add(client);
}
}
@ -61,21 +80,100 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient {
* Retry failed DBs
* @param msg is the message,
* @return the message ID for later retrieval
* @throws CommunicationException
*/
@Override
public MessageID postMessage(BulletinBoardMessage msg, ClientCallback<?> callback){
public MessageID postMessage(BulletinBoardMessage msg, FutureCallback<Boolean> callback){
// Create job
BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.POST_MESSAGE, msg, -1);
MultiServerPostMessageWorker worker =
new MultiServerPostMessageWorker(clients, minAbsoluteRedundancy, msg, POST_MESSAGE_RETRY_NUM, callback);
// Submit job and create callback
Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new PostMessageFutureCallback(listeningExecutor, callback));
// Submit job
executorService.submit(worker);
// Calculate the correct message ID and return it
digest.reset();
digest.update(msg.getMsg());
return MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build();
batchDigest.reset();
batchDigest.update(msg.getMsg());
return batchDigest.digestAsMessageID();
}
@Override
public MessageID postBatch(CompleteBatch completeBatch, FutureCallback<Boolean> callback) {
// Create job
MultiServerPostBatchWorker worker =
new MultiServerPostBatchWorker(clients, minAbsoluteRedundancy, completeBatch, POST_MESSAGE_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
// Calculate the correct message ID and return it
batchDigest.reset();
batchDigest.update(completeBatch);
return batchDigest.digestAsMessageID();
}
@Override
public void beginBatch(BeginBatchMessage beginBatchMessage, FutureCallback<Boolean> callback) {
// Create job
MultiServerBeginBatchWorker worker =
new MultiServerBeginBatchWorker(clients, minAbsoluteRedundancy, beginBatchMessage, POST_MESSAGE_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
}
@Override
public void postBatchData(byte[] signerId, int batchId, List<BatchData> batchDataList,
int startPosition, FutureCallback<Boolean> callback) {
BatchDataContainer batchDataContainer = new BatchDataContainer(signerId, batchId, batchDataList, startPosition);
// Create job
MultiServerPostBatchDataWorker worker =
new MultiServerPostBatchDataWorker(clients, minAbsoluteRedundancy, batchDataContainer, POST_MESSAGE_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
}
@Override
public void postBatchData(byte[] signerId, int batchId, List<BatchData> batchDataList, FutureCallback<Boolean> callback) {
postBatchData(signerId, batchId, batchDataList, 0, callback);
}
@Override
public void postBatchData(ByteString signerId, int batchId, List<BatchData> batchDataList,
int startPosition, FutureCallback<Boolean> callback) {
postBatchData(signerId.toByteArray(), batchId, batchDataList, startPosition, callback);
}
@Override
public void postBatchData(ByteString signerId, int batchId, List<BatchData> batchDataList, FutureCallback<Boolean> callback) {
postBatchData(signerId, batchId, batchDataList, 0, callback);
}
@Override
public void closeBatch(CloseBatchMessage closeBatchMessage, FutureCallback<Boolean> callback) {
// Create job
MultiServerCloseBatchWorker worker =
new MultiServerCloseBatchWorker(clients, minAbsoluteRedundancy, closeBatchMessage, POST_MESSAGE_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
}
/**
@ -83,17 +181,16 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient {
* Return the number of databases in which the message was found
* Only try once per DB
* Ignore communication exceptions in specific databases
* @param id is the requested message ID
* @return the number of DBs in which retrieval was successful
*/
@Override
public void getRedundancy(MessageID id, ClientCallback<Float> callback) {
public void getRedundancy(MessageID id, FutureCallback<Float> callback) {
// Create job
BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.GET_REDUNDANCY, id, 1);
MultiServerGetRedundancyWorker worker =
new MultiServerGetRedundancyWorker(clients, minAbsoluteRedundancy, id, GET_REDUNDANCY_RETRY_NUM, callback);
// Submit job and create callback
Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new GetRedundancyFutureCallback(listeningExecutor, callback));
// Submit job
executorService.submit(worker);
}
@ -101,27 +198,49 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient {
* Go through the DBs and try to retrieve messages according to the specified filter
* If at the operation is successful for some DB: return the results and stop iterating
* If no operation is successful: return null (NOT blank list)
* @param filterList return only messages that match the filters (null means no filtering).
* @return
*/
@Override
public void readMessages(MessageFilterList filterList, ClientCallback<List<BulletinBoardMessage>> callback) {
public void readMessages(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
// Create job
BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.READ_MESSAGES,
filterList, READ_MESSAGES_RETRY_NUM);
MultiServerReadMessagesWorker worker =
new MultiServerReadMessagesWorker(clients, minAbsoluteRedundancy, filterList, READ_MESSAGES_RETRY_NUM, callback);
// Submit job and create callback
Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new ReadMessagesFutureCallback(listeningExecutor, callback));
// Submit job
executorService.submit(worker);
}
@Override
public void readBatch(BatchSpecificationMessage batchSpecificationMessage, FutureCallback<CompleteBatch> callback) {
// Create job
MultiServerReadBatchWorker worker =
new MultiServerReadBatchWorker(clients, minAbsoluteRedundancy, batchSpecificationMessage, READ_MESSAGES_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
}
@Override
public void subscribe(MessageFilterList filterList, MessageHandler messageHandler) {
// TODO: Implement
}
@Override
public void close() {
super.close();
try {
listeningExecutor.shutdown();
while (! listeningExecutor.isShutdown()) {
listeningExecutor.awaitTermination(10, TimeUnit.SECONDS);
for (SingleServerBulletinBoardClient client : clients){
client.close();
}
executorService.shutdown();
while (! executorService.isShutdown()) {
executorService.awaitTermination(10, TimeUnit.SECONDS);
}
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());

25
bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java
View File

@ -1,25 +0,0 @@
package meerkat.bulletinboard.callbacks;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListeningExecutorService;
import meerkat.bulletinboard.BulletinClientJob;
import meerkat.bulletinboard.BulletinClientJobResult;
import meerkat.bulletinboard.BulletinClientWorker;
import meerkat.protobuf.BulletinBoardAPI;
import java.util.List;
/**
* This is a future callback used to listen to workers and run on job finish
* Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error
*/
public abstract class ClientFutureCallback implements FutureCallback<BulletinClientJobResult> {
protected ListeningExecutorService listeningExecutor;
ClientFutureCallback(ListeningExecutorService listeningExecutor) {
this.listeningExecutor = listeningExecutor;
}
}

38
bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java
View File

@ -1,38 +0,0 @@
package meerkat.bulletinboard.callbacks;
import com.google.common.util.concurrent.ListeningExecutorService;
import meerkat.bulletinboard.BulletinBoardClient;
import meerkat.bulletinboard.BulletinClientJobResult;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.List;
/**
* This is a future callback used to listen to workers and run on job finish
* Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error
*/
public class GetRedundancyFutureCallback extends ClientFutureCallback {
private BulletinBoardClient.ClientCallback<Float> callback;
public GetRedundancyFutureCallback(ListeningExecutorService listeningExecutor,
BulletinBoardClient.ClientCallback<Float> callback) {
super(listeningExecutor);
this.callback = callback;
}
@Override
public void onSuccess(BulletinClientJobResult result) {
int absoluteRedundancy = ((IntMsg) result.getResult()).getValue();
int totalServers = result.getJob().getServerAddresses().size();
callback.handleCallback( ((float) absoluteRedundancy) / ((float) totalServers) );
}
@Override
public void onFailure(Throwable t) {
callback.handleFailure(t);
}
}

46
bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java
View File

@ -1,46 +0,0 @@
package meerkat.bulletinboard.callbacks;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListeningExecutorService;
import meerkat.bulletinboard.BulletinBoardClient;
import meerkat.bulletinboard.BulletinClientJob;
import meerkat.bulletinboard.BulletinClientJobResult;
import meerkat.bulletinboard.BulletinClientWorker;
import meerkat.protobuf.BulletinBoardAPI;
import java.util.List;
/**
* This is a future callback used to listen to workers and run on job finish
* Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error
*/
public class PostMessageFutureCallback extends ClientFutureCallback {
private BulletinBoardClient.ClientCallback<?> callback;
public PostMessageFutureCallback(ListeningExecutorService listeningExecutor,
BulletinBoardClient.ClientCallback<?> callback) {
super(listeningExecutor);
this.callback = callback;
}
@Override
public void onSuccess(BulletinClientJobResult result) {
BulletinClientJob job = result.getJob();
job.decMaxRetry();
// If redundancy is below threshold: retry
if (job.getMinServers() > 0 && job.isRetry()) {
Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), this);
}
callback.handleCallback(null);
}
@Override
public void onFailure(Throwable t) {
callback.handleFailure(t);
}
}

38
bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java
View File

@ -1,38 +0,0 @@
package meerkat.bulletinboard.callbacks;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListeningExecutorService;
import meerkat.bulletinboard.BulletinBoardClient;
import meerkat.bulletinboard.BulletinClientJob;
import meerkat.bulletinboard.BulletinClientJobResult;
import meerkat.bulletinboard.BulletinClientWorker;
import meerkat.protobuf.BulletinBoardAPI;
import java.util.List;
/**
* This is a future callback used to listen to workers and run on job finish
* Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error
*/
public class ReadMessagesFutureCallback extends ClientFutureCallback {
private BulletinBoardClient.ClientCallback<List<BulletinBoardAPI.BulletinBoardMessage>> callback;
public ReadMessagesFutureCallback(ListeningExecutorService listeningExecutor,
BulletinBoardClient.ClientCallback<List<BulletinBoardAPI.BulletinBoardMessage>> callback) {
super(listeningExecutor);
this.callback = callback;
}
@Override
public void onSuccess(BulletinClientJobResult result) {
callback.handleCallback(((BulletinBoardAPI.BulletinBoardMessageList) result.getResult()).getMessageList());
}
@Override
public void onFailure(Throwable t) {
callback.handleFailure(t);
}
}

28
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java Normal file
View File

@ -0,0 +1,28 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerBeginBatchWorker extends MultiServerGenericPostWorker<BeginBatchMessage> {
public MultiServerBeginBatchWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, BeginBatchMessage payload, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doPost(SingleServerBulletinBoardClient client, BeginBatchMessage payload) {
client.beginBatch(payload, this);
}
}

28
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java Normal file
View File

@ -0,0 +1,28 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.CloseBatchMessage;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerCloseBatchWorker extends MultiServerGenericPostWorker<CloseBatchMessage> {
public MultiServerCloseBatchWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, CloseBatchMessage payload, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doPost(SingleServerBulletinBoardClient client, CloseBatchMessage payload) {
client.closeBatch(payload, this);
}
}

67
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java Normal file
View File

@ -0,0 +1,67 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.MultiServerWorker;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.comm.CommunicationException;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import java.util.Iterator;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public abstract class MultiServerGenericPostWorker<T> extends MultiServerWorker<T, Boolean> {
public MultiServerGenericPostWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, T payload, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
protected abstract void doPost(SingleServerBulletinBoardClient client, T payload);
/**
* This method carries out the actual communication with the servers via HTTP Post
* It accesses the servers one by one and tries to post the payload to each in turn
* The method will only iterate once through the server list
* Successful post to a server results in removing the server from the list
*/
public void run() {
// Iterate through servers
Iterator<SingleServerBulletinBoardClient> clientIterator = getClientIterator();
while (clientIterator.hasNext()) {
// Send request to Server
SingleServerBulletinBoardClient client = clientIterator.next();
doPost(client, payload);
}
}
@Override
public void onSuccess(Boolean result) {
if (result){
if (minServers.decrementAndGet() <= 0){
succeed(Boolean.TRUE);
}
}
}
@Override
public void onFailure(Throwable t) {
if (maxFailedServers.decrementAndGet() < 0){
fail(t);
}
}
}

64
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java Normal file
View File

@ -0,0 +1,64 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.MultiServerWorker;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.comm.CommunicationException;
import java.util.Iterator;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public abstract class MultiServerGenericReadWorker<IN, OUT> extends MultiServerWorker<IN, OUT>{
private final Iterator<SingleServerBulletinBoardClient> clientIterator;
public MultiServerGenericReadWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, IN payload, int maxRetry,
FutureCallback<OUT> futureCallback) {
super(clients, true, minServers, payload, maxRetry, futureCallback); // Shuffle clients on creation to balance load
clientIterator = getClientIterator();
}
protected abstract void doRead(IN payload, SingleServerBulletinBoardClient client);
/**
* This method carries out the actual communication with the servers via HTTP Post
* It accesses the servers in a random order until one answers it
* Successful retrieval from any server terminates the method and returns the received values; The list is not changed
*/
public void run(){
// Iterate through servers
if (clientIterator.hasNext()) {
// Get next server
SingleServerBulletinBoardClient client = clientIterator.next();
// Retrieve answer from server
doRead(payload, client);
} else {
fail(new CommunicationException("Could not contact any server"));
}
}
@Override
public void onSuccess(OUT msg) {
succeed(msg);
}
@Override
public void onFailure(Throwable t) {
run(); // Retry with next server
}
}

72
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java Normal file
View File

@ -0,0 +1,72 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.MultiServerWorker;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerGetRedundancyWorker extends MultiServerWorker<MessageID, Float> {
private AtomicInteger serversContainingMessage;
private AtomicInteger totalContactedServers;
public MultiServerGetRedundancyWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, MessageID payload, int maxRetry,
FutureCallback<Float> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback); // Shuffle clients on creation to balance load
serversContainingMessage = new AtomicInteger(0);
totalContactedServers = new AtomicInteger(0);
}
/**
* This method carries out the actual communication with the servers via HTTP Post
* It accesses the servers in a random order until one answers it
* Successful retrieval from any server terminates the method and returns the received values; The list is not changed
*/
public void run(){
Iterator<SingleServerBulletinBoardClient> clientIterator = getClientIterator();
// Iterate through clients
while (clientIterator.hasNext()) {
SingleServerBulletinBoardClient client = clientIterator.next();
// Send request to client
client.getRedundancy(payload,this);
}
}
@Override
public void onSuccess(Float result) {
if (result > 0.5) {
serversContainingMessage.incrementAndGet();
}
if (totalContactedServers.incrementAndGet() >= getClientNumber()){
succeed(((float) serversContainingMessage.get()) / ((float) getClientNumber()));
}
}
@Override
public void onFailure(Throwable t) {
onSuccess(0.0f);
}
}

28
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchDataWorker.java Normal file
View File

@ -0,0 +1,28 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.bulletinboard.BatchDataContainer;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerPostBatchDataWorker extends MultiServerGenericPostWorker<BatchDataContainer> {
public MultiServerPostBatchDataWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, BatchDataContainer payload, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doPost(SingleServerBulletinBoardClient client, BatchDataContainer payload) {
client.postBatchData(payload.signerId, payload.batchId, payload.batchDataList, payload.startPosition, this);
}
}

28
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java Normal file
View File

@ -0,0 +1,28 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.CompleteBatch;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerPostBatchWorker extends MultiServerGenericPostWorker<CompleteBatch> {
public MultiServerPostBatchWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, CompleteBatch payload, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doPost(SingleServerBulletinBoardClient client, CompleteBatch payload) {
client.postBatch(payload, this);
}
}

28
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java Normal file
View File

@ -0,0 +1,28 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerPostMessageWorker extends MultiServerGenericPostWorker<BulletinBoardMessage> {
public MultiServerPostMessageWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, BulletinBoardMessage payload, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doPost(SingleServerBulletinBoardClient client, BulletinBoardMessage payload) {
client.postMessage(payload, this);
}
}

30
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchWorker.java Normal file
View File

@ -0,0 +1,30 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.CompleteBatch;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.BatchSpecificationMessage;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerReadBatchWorker extends MultiServerGenericReadWorker<BatchSpecificationMessage, CompleteBatch> {
public MultiServerReadBatchWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, BatchSpecificationMessage payload, int maxRetry,
FutureCallback<CompleteBatch> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doRead(BatchSpecificationMessage payload, SingleServerBulletinBoardClient client) {
client.readBatch(payload, this);
}
}

29
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java Normal file
View File

@ -0,0 +1,29 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerReadMessagesWorker extends MultiServerGenericReadWorker<MessageFilterList,List<BulletinBoardMessage>>{
public MultiServerReadMessagesWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, MessageFilterList payload, int maxRetry,
FutureCallback<List<BulletinBoardMessage>> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doRead(MessageFilterList payload, SingleServerBulletinBoardClient client) {
client.readMessages(payload, this);
}
}

17
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerBeginBatchWorker.java Normal file
View File

@ -0,0 +1,17 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage;
import static meerkat.bulletinboard.BulletinBoardConstants.BEGIN_BATCH_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a post operation
*/
public class SingleServerBeginBatchWorker extends SingleServerGenericPostWorker<BeginBatchMessage> {
public SingleServerBeginBatchWorker(String serverAddress, BeginBatchMessage payload, int maxRetry) {
super(serverAddress, BEGIN_BATCH_PATH, payload, maxRetry);
}
}

17
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerCloseBatchWorker.java Normal file
View File

@ -0,0 +1,17 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.protobuf.BulletinBoardAPI.CloseBatchMessage;
import static meerkat.bulletinboard.BulletinBoardConstants.CLOSE_BATCH_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a close batch operation
*/
public class SingleServerCloseBatchWorker extends SingleServerGenericPostWorker<CloseBatchMessage> {
public SingleServerCloseBatchWorker(String serverAddress, CloseBatchMessage payload, int maxRetry) {
super(serverAddress, CLOSE_BATCH_PATH, payload, maxRetry);
}
}

62
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java Normal file
View File

@ -0,0 +1,62 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.BoolMsg;
import meerkat.rest.Constants;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a post operation
*/
public class SingleServerGenericPostWorker<T> extends SingleServerWorker<T, Boolean> {
private final String subPath;
public SingleServerGenericPostWorker(String serverAddress, String subPath, T payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
this.subPath = subPath;
}
/**
* This method carries out the actual communication with the server via HTTP Post
* It accesses the server and tries to post the payload to it
* Successful post to a server results
* @return TRUE if the operation is successful
* @throws CommunicationException if the operation is unsuccessful
*/
public Boolean call() throws CommunicationException{
Client client = clientLocal.get();
WebTarget webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(subPath);
Response response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(
Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF));
try {
// If a BoolMsg entity is returned: the post was successful
response.readEntity(BoolMsg.class);
return Boolean.TRUE;
} catch (ProcessingException | IllegalStateException e) {
// Post to this server failed
throw new CommunicationException("Could not contact the server");
}
finally {
response.close();
}
}
}

79
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGetRedundancyWorker.java Normal file
View File

@ -0,0 +1,79 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.rest.Constants;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class SingleServerGetRedundancyWorker extends SingleServerWorker<MessageID, Float> {
public SingleServerGetRedundancyWorker(String serverAddress, MessageID payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
}
/**
* This method carries out the actual communication with the server via HTTP Post
* It queries the server for a message with the given ID
* @return TRUE if the message exists in the server and FALSE otherwise
* @throws CommunicationException if the server does not return a valid answer
*/
public Float call() throws CommunicationException{
Client client = clientLocal.get();
WebTarget webTarget;
Response response;
MessageFilterList msgFilterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(payload.getID())
.build()
).build();
// Send request to Server
webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msgFilterList, Constants.MEDIATYPE_PROTOBUF));
// Retrieve answer
try {
// If a BulletinBoardMessageList is returned: the read was successful
BulletinBoardMessageList msgList = response.readEntity(BulletinBoardMessageList.class);
if (msgList.getMessageList().size() > 0){
// Message exists in the server
return 1.0f;
}
else {
// Message does not exist in the server
return 0.0f;
}
} catch (ProcessingException | IllegalStateException e) {
// Read failed
throw new CommunicationException("Server access failed");
}
finally {
response.close();
}
}
}

17
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostBatchWorker.java Normal file
View File

@ -0,0 +1,17 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.protobuf.BulletinBoardAPI.BatchMessage;
import static meerkat.bulletinboard.BulletinBoardConstants.POST_BATCH_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a post batch operation
*/
public class SingleServerPostBatchWorker extends SingleServerGenericPostWorker<BatchMessage> {
public SingleServerPostBatchWorker(String serverAddress, BatchMessage payload, int maxRetry) {
super(serverAddress, POST_BATCH_PATH, payload, maxRetry);
}
}

17
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostMessageWorker.java Normal file
View File

@ -0,0 +1,17 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import static meerkat.bulletinboard.BulletinBoardConstants.POST_MESSAGE_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a post operation
*/
public class SingleServerPostMessageWorker extends SingleServerGenericPostWorker<BulletinBoardMessage> {
public SingleServerPostMessageWorker(String serverAddress, BulletinBoardMessage payload, int maxRetry) {
super(serverAddress, POST_MESSAGE_PATH, payload, maxRetry);
}
}

78
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java Normal file
View File

@ -0,0 +1,78 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.bulletinboard.CompleteBatch;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageInputStream;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.rest.Constants;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.READ_BATCH_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.BATCH_ID_TAG_PREFIX;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class SingleServerReadBatchWorker extends SingleServerWorker<BatchSpecificationMessage, List<BatchData>> {
public SingleServerReadBatchWorker(String serverAddress, BatchSpecificationMessage payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
}
/**
* This method carries out the actual communication with the server via HTTP Post
* Upon successful retrieval from the server the method returns the received values
* @return the complete batch as read from the server
* @throws CommunicationException if the server's response is invalid
*/
public List<BatchData> call() throws CommunicationException{
Client client = clientLocal.get();
WebTarget webTarget;
// Get the batch data
webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_BATCH_PATH);
InputStream in = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF), InputStream.class);
MessageInputStream<BatchData> inputStream = null;
try {
inputStream = MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BatchData.class);
return inputStream.asList();
} catch (IOException | InvocationTargetException e) {
// Read failed
throw new CommunicationException("Could not contact the server or server returned illegal result");
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new CommunicationException("MessageInputStream error");
} finally {
try {
inputStream.close();
} catch (IOException ignored) {}
}
}
}

76
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java Normal file
View File

@ -0,0 +1,76 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageInputStream;
import meerkat.protobuf.BulletinBoardAPI;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessageList;
import meerkat.protobuf.BulletinBoardAPI.MessageFilterList;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.rest.Constants;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class SingleServerReadMessagesWorker extends SingleServerWorker<MessageFilterList, List<BulletinBoardMessage>> {
public SingleServerReadMessagesWorker(String serverAddress, MessageFilterList payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
}
/**
* This method carries out the actual communication with the server via HTTP Post
* Upon successful retrieval from the server the method returns the received values
* @return The list of messages returned by the server
* @throws CommunicationException if the server's response is invalid
*/
public List<BulletinBoardMessage> call() throws CommunicationException{
Client client = clientLocal.get();
WebTarget webTarget;
// Send request to Server
webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH);
InputStream in = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF), InputStream.class);
MessageInputStream<BulletinBoardMessage> inputStream = null;
try {
inputStream = MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BulletinBoardMessage.class);
return inputStream.asList();
} catch (IOException | InvocationTargetException e) {
// Read failed
throw new CommunicationException("Could not contact the server or server returned illegal result");
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new CommunicationException("MessageInputStream error");
} finally {
try {
inputStream.close();
} catch (IOException ignored) {}
}
}
}

212
bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java
View File

@ -1,212 +0,0 @@
import com.google.protobuf.ByteString;
import meerkat.bulletinboard.BulletinBoardClient;
import meerkat.bulletinboard.BulletinBoardClient.ClientCallback;
import meerkat.bulletinboard.ThreadedBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto;
import meerkat.protobuf.Voting.BulletinBoardClientParams;
import meerkat.util.BulletinBoardMessageComparator;
import org.junit.Before;
import org.junit.Test;
import java.util.*;
import java.util.concurrent.Semaphore;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.number.OrderingComparison.greaterThanOrEqualTo;
import static org.junit.Assert.assertThat;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
*/
public class BulletinBoardClientIntegrationTest {
Semaphore jobSemaphore;
Vector<Throwable> thrown;
private class PostCallback implements ClientCallback<Object>{
@Override
public void handleCallback(Object msg) {
System.err.println("Post operation completed");
jobSemaphore.release();
}
@Override
public void handleFailure(Throwable t) {
thrown.add(t);
jobSemaphore.release();
}
}
private class RedundancyCallback implements ClientCallback<Float>{
private float minRedundancy;
public RedundancyCallback(float minRedundancy) {
this.minRedundancy = minRedundancy;
}
@Override
public void handleCallback(Float redundancy) {
System.err.println("Redundancy found is: " + redundancy);
jobSemaphore.release();
assertThat(redundancy, greaterThanOrEqualTo(minRedundancy));
}
@Override
public void handleFailure(Throwable t) {
thrown.add(t);
jobSemaphore.release();
}
}
private class ReadCallback implements ClientCallback<List<BulletinBoardMessage>>{
private List<BulletinBoardMessage> expectedMsgList;
public ReadCallback(List<BulletinBoardMessage> expectedMsgList) {
this.expectedMsgList = expectedMsgList;
}
@Override
public void handleCallback(List<BulletinBoardMessage> messages) {
System.err.println(messages);
jobSemaphore.release();
BulletinBoardMessageComparator msgComparator = new BulletinBoardMessageComparator();
assertThat(messages.size(), is(expectedMsgList.size()));
Iterator<BulletinBoardMessage> expectedMessageIterator = expectedMsgList.iterator();
Iterator<BulletinBoardMessage> receivedMessageIterator = messages.iterator();
while (expectedMessageIterator.hasNext()) {
assertThat(msgComparator.compare(expectedMessageIterator.next(), receivedMessageIterator.next()), is(0));
}
}
@Override
public void handleFailure(Throwable t) {
thrown.add(t);
jobSemaphore.release();
}
}
private BulletinBoardClient bulletinBoardClient;
private PostCallback postCallback;
private RedundancyCallback redundancyCallback;
private ReadCallback readCallback;
private static String PROP_GETTY_URL = "gretty.httpBaseURI";
private static String DEFAULT_BASE_URL = "http://localhost:8081";
private static String BASE_URL = System.getProperty(PROP_GETTY_URL, DEFAULT_BASE_URL);
@Before
public void init(){
bulletinBoardClient = new ThreadedBulletinBoardClient();
List<String> testDB = new LinkedList<String>();
testDB.add(BASE_URL);
bulletinBoardClient.init(BulletinBoardClientParams.newBuilder()
.addBulletinBoardAddress("http://localhost:8081")
.setMinRedundancy((float) 1.0)
.build());
postCallback = new PostCallback();
redundancyCallback = new RedundancyCallback((float) 1.0);
thrown = new Vector<>();
jobSemaphore = new Semaphore(0);
}
@Test
public void postTest() {
byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4};
byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14};
byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24};
byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0};
BulletinBoardMessage msg;
MessageFilterList filterList;
List<BulletinBoardMessage> msgList;
MessageID messageID;
Comparator<BulletinBoardMessage> msgComparator = new BulletinBoardMessageComparator();
msg = BulletinBoardMessage.newBuilder()
.setMsg(UnsignedBulletinBoardMessage.newBuilder()
.addTag("Signature")
.addTag("Trustee")
.setData(ByteString.copyFrom(b1))
.build())
.addSig(Crypto.Signature.newBuilder()
.setType(Crypto.SignatureType.DSA)
.setData(ByteString.copyFrom(b2))
.setSignerId(ByteString.copyFrom(b3))
.build())
.addSig(Crypto.Signature.newBuilder()
.setType(Crypto.SignatureType.ECDSA)
.setData(ByteString.copyFrom(b3))
.setSignerId(ByteString.copyFrom(b2))
.build())
.build();
messageID = bulletinBoardClient.postMessage(msg,postCallback);
try {
jobSemaphore.acquire();
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());
}
bulletinBoardClient.getRedundancy(messageID,redundancyCallback);
filterList = MessageFilterList.newBuilder()
.addFilter(
MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag("Signature")
.build()
)
.addFilter(
MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag("Trustee")
.build()
)
.build();
msgList = new LinkedList<BulletinBoardMessage>();
msgList.add(msg);
readCallback = new ReadCallback(msgList);
bulletinBoardClient.readMessages(filterList, readCallback);
try {
jobSemaphore.acquire(2);
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());
}
bulletinBoardClient.close();
for (Throwable t : thrown) {
System.err.println(t.getMessage());
}
if (thrown.size() > 0) {
assert false;
}
}
}

556
bulletin-board-client/src/test/java/ThreadedBulletinBoardClientIntegrationTest.java Normal file
View File

@ -0,0 +1,556 @@
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.*;
import com.google.protobuf.Timestamp;
import meerkat.bulletinboard.AsyncBulletinBoardClient;
import meerkat.bulletinboard.CompleteBatch;
import meerkat.bulletinboard.GenericBatchDigitalSignature;
import meerkat.bulletinboard.ThreadedBulletinBoardClient;
import meerkat.comm.CommunicationException;
import meerkat.crypto.concrete.ECDSASignature;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto;
import meerkat.protobuf.Voting.*;
import meerkat.util.BulletinBoardMessageComparator;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.number.OrderingComparison.*;
import java.io.IOException;
import java.io.InputStream;
import java.security.*;
import java.security.cert.CertificateException;
import java.util.*;
import java.util.concurrent.Semaphore;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
*/
public class ThreadedBulletinBoardClientIntegrationTest {
// Signature resources
private GenericBatchDigitalSignature signers[];
private ByteString[] signerIDs;
private static String KEYFILE_EXAMPLE = "/certs/enduser-certs/user1-key-with-password-secret.p12";
private static String KEYFILE_EXAMPLE3 = "/certs/enduser-certs/user3-key-with-password-shh.p12";
private static String KEYFILE_PASSWORD1 = "secret";
private static String KEYFILE_PASSWORD3 = "shh";
private static String CERT1_PEM_EXAMPLE = "/certs/enduser-certs/user1.crt";
private static String CERT3_PEM_EXAMPLE = "/certs/enduser-certs/user3.crt";
// Server data
private static String PROP_GETTY_URL = "gretty.httpBaseURI";
private static String DEFAULT_BASE_URL = "http://localhost:8081";
private static String BASE_URL = System.getProperty(PROP_GETTY_URL, DEFAULT_BASE_URL);
// Client and callbacks
private AsyncBulletinBoardClient bulletinBoardClient;
private PostCallback postCallback;
private PostCallback failPostCallback = new PostCallback(true,false);
private RedundancyCallback redundancyCallback;
private ReadCallback readCallback;
private ReadBatchCallback readBatchCallback;
// Sync and misc
private Semaphore jobSemaphore;
private Vector<Throwable> thrown;
private Random random;
// Constructor
public ThreadedBulletinBoardClientIntegrationTest(){
signers = new GenericBatchDigitalSignature[2];
signerIDs = new ByteString[signers.length];
signers[0] = new GenericBatchDigitalSignature(new ECDSASignature());
signers[1] = new GenericBatchDigitalSignature(new ECDSASignature());
InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE);
char[] password = KEYFILE_PASSWORD1.toCharArray();
KeyStore.Builder keyStoreBuilder;
try {
keyStoreBuilder = signers[0].getPKCS12KeyStoreBuilder(keyStream, password);
signers[0].loadSigningCertificate(keyStoreBuilder);
signers[0].loadVerificationCertificates(getClass().getResourceAsStream(CERT1_PEM_EXAMPLE));
keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE3);
password = KEYFILE_PASSWORD3.toCharArray();
keyStoreBuilder = signers[1].getPKCS12KeyStoreBuilder(keyStream, password);
signers[1].loadSigningCertificate(keyStoreBuilder);
signers[1].loadVerificationCertificates(getClass().getResourceAsStream(CERT3_PEM_EXAMPLE));
for (int i = 0 ; i < signers.length ; i++) {
signerIDs[i] = signers[i].getSignerID();
}
} catch (IOException e) {
System.err.println("Failed reading from signature file " + e.getMessage());
fail("Failed reading from signature file " + e.getMessage());
} catch (CertificateException e) {
System.err.println("Failed reading certificate " + e.getMessage());
fail("Failed reading certificate " + e.getMessage());
} catch (KeyStoreException e) {
System.err.println("Failed reading keystore " + e.getMessage());
fail("Failed reading keystore " + e.getMessage());
} catch (NoSuchAlgorithmException e) {
System.err.println("Couldn't find signing algorithm " + e.getMessage());
fail("Couldn't find signing algorithm " + e.getMessage());
} catch (UnrecoverableKeyException e) {
System.err.println("Couldn't find signing key " + e.getMessage());
fail("Couldn't find signing key " + e.getMessage());
}
}
// Callback definitions
protected void genericHandleFailure(Throwable t){
System.err.println(t.getCause() + " " + t.getMessage());
thrown.add(t);
jobSemaphore.release();
}
private class PostCallback implements FutureCallback<Boolean>{
private boolean isAssert;
private boolean assertValue;
public PostCallback() {
this(false);
}
public PostCallback(boolean isAssert) {
this(isAssert,true);
}
public PostCallback(boolean isAssert, boolean assertValue) {
this.isAssert = isAssert;
this.assertValue = assertValue;
}
@Override
public void onSuccess(Boolean msg) {
System.err.println("Post operation completed");
jobSemaphore.release();
//TODO: Change Assert mechanism to exception one
if (isAssert) {
if (assertValue) {
assertThat("Post operation failed", msg, is(Boolean.TRUE));
} else {
assertThat("Post operation succeeded unexpectedly", msg, is(Boolean.FALSE));
}
}
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
}
private class RedundancyCallback implements FutureCallback<Float>{
private float minRedundancy;
public RedundancyCallback(float minRedundancy) {
this.minRedundancy = minRedundancy;
}
@Override
public void onSuccess(Float redundancy) {
System.err.println("Redundancy found is: " + redundancy);
jobSemaphore.release();
assertThat(redundancy, greaterThanOrEqualTo(minRedundancy));
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
}
private class ReadCallback implements FutureCallback<List<BulletinBoardMessage>>{
private List<BulletinBoardMessage> expectedMsgList;
public ReadCallback(List<BulletinBoardMessage> expectedMsgList) {
this.expectedMsgList = expectedMsgList;
}
@Override
public void onSuccess(List<BulletinBoardMessage> messages) {
System.err.println(messages);
jobSemaphore.release();
BulletinBoardMessageComparator msgComparator = new BulletinBoardMessageComparator();
assertThat(messages.size(), is(expectedMsgList.size()));
Iterator<BulletinBoardMessage> expectedMessageIterator = expectedMsgList.iterator();
Iterator<BulletinBoardMessage> receivedMessageIterator = messages.iterator();
while (expectedMessageIterator.hasNext()) {
assertThat(msgComparator.compare(expectedMessageIterator.next(), receivedMessageIterator.next()), is(0));
}
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
}
private class ReadBatchCallback implements FutureCallback<CompleteBatch> {
private CompleteBatch expectedBatch;
public ReadBatchCallback(CompleteBatch expectedBatch) {
this.expectedBatch = expectedBatch;
}
@Override
public void onSuccess(CompleteBatch batch) {
System.err.println(batch);
jobSemaphore.release();
assertThat("Batch returned is incorrect", batch, is(equalTo(expectedBatch)));
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
}
// Randomness generators
private byte randomByte(){
return (byte) random.nextInt();
}
private byte[] randomByteArray(int length) {
byte[] randomBytes = new byte[length];
for (int i = 0; i < length ; i++){
randomBytes[i] = randomByte();
}
return randomBytes;
}
private CompleteBatch createRandomBatch(int signer, int batchId, int length) throws SignatureException {
CompleteBatch completeBatch = new CompleteBatch();
// Create data
completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder()
.setSignerId(signerIDs[signer])
.setBatchId(batchId)
.addTag("Test")
.build());
for (int i = 0 ; i < length ; i++){
BatchData batchData = BatchData.newBuilder()
.setData(ByteString.copyFrom(randomByteArray(i)))
.build();
completeBatch.appendBatchData(batchData);
}
completeBatch.setTimestamp(Timestamp.newBuilder()
.setSeconds(Math.abs(90))
.setNanos(50)
.build());
signers[signer].updateContent(completeBatch);
completeBatch.setSignature(signers[signer].sign());
return completeBatch;
}
// Test methods
/**
* Takes care of initializing the client and the test resources
*/
@Before
public void init(){
bulletinBoardClient = new ThreadedBulletinBoardClient();
random = new Random(0); // We use insecure randomness in tests for repeatability
List<String> testDB = new LinkedList<>();
testDB.add(BASE_URL);
bulletinBoardClient.init(BulletinBoardClientParams.newBuilder()
.addAllBulletinBoardAddress(testDB)
.setMinRedundancy((float) 1.0)
.build());
postCallback = new PostCallback();
redundancyCallback = new RedundancyCallback((float) 1.0);
thrown = new Vector<>();
jobSemaphore = new Semaphore(0);
}
/**
* Closes the client and makes sure the test fails when an exception occurred in a separate thread
*/
@After
public void close() {
bulletinBoardClient.close();
if (thrown.size() > 0) {
assert false;
}
}
/**
* Tests the standard post, redundancy and read methods
*/
@Test
public void postTest() {
byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4};
byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14};
byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24};
BulletinBoardMessage msg;
MessageFilterList filterList;
List<BulletinBoardMessage> msgList;
MessageID messageID;
msg = BulletinBoardMessage.newBuilder()
.setMsg(UnsignedBulletinBoardMessage.newBuilder()
.addTag("Signature")
.addTag("Trustee")
.setData(ByteString.copyFrom(b1))
.setTimestamp(Timestamp.newBuilder()
.setSeconds(20)
.setNanos(30)
.build())
.build())
.addSig(Crypto.Signature.newBuilder()
.setType(Crypto.SignatureType.DSA)
.setData(ByteString.copyFrom(b2))
.setSignerId(ByteString.copyFrom(b3))
.build())
.addSig(Crypto.Signature.newBuilder()
.setType(Crypto.SignatureType.ECDSA)
.setData(ByteString.copyFrom(b3))
.setSignerId(ByteString.copyFrom(b2))
.build())
.build();
messageID = bulletinBoardClient.postMessage(msg,postCallback);
try {
jobSemaphore.acquire();
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());
}
bulletinBoardClient.getRedundancy(messageID,redundancyCallback);
filterList = MessageFilterList.newBuilder()
.addFilter(
MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag("Signature")
.build()
)
.addFilter(
MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag("Trustee")
.build()
)
.build();
msgList = new LinkedList<>();
msgList.add(msg);
readCallback = new ReadCallback(msgList);
bulletinBoardClient.readMessages(filterList, readCallback);
try {
jobSemaphore.acquire(2);
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());
}
}
/**
* Tests posting a batch by parts
* Also tests not being able to post to a closed batch
* @throws CommunicationException, SignatureException, InterruptedException
*/
@Test
public void testBatchPost() throws CommunicationException, SignatureException, InterruptedException {
final int SIGNER = 1;
final int BATCH_ID = 100;
final int BATCH_LENGTH = 100;
CompleteBatch completeBatch = createRandomBatch(SIGNER, BATCH_ID, BATCH_LENGTH);
// Begin batch
bulletinBoardClient.beginBatch(completeBatch.getBeginBatchMessage(), postCallback);
jobSemaphore.acquire();
// Post data
bulletinBoardClient.postBatchData(signerIDs[SIGNER], BATCH_ID, completeBatch.getBatchDataList(), postCallback);
jobSemaphore.acquire();
// Close batch
CloseBatchMessage closeBatchMessage = CloseBatchMessage.newBuilder()
.setBatchId(BATCH_ID)
.setBatchLength(BATCH_LENGTH)
.setTimestamp(Timestamp.newBuilder()
.setSeconds(50)
.setNanos(80)
.build())
.setSig(completeBatch.getSignature())
.build();
bulletinBoardClient.closeBatch(closeBatchMessage, postCallback);
jobSemaphore.acquire();
// Attempt to open batch again
bulletinBoardClient.beginBatch(completeBatch.getBeginBatchMessage(), failPostCallback);
// Attempt to add batch data
bulletinBoardClient.postBatchData(signerIDs[SIGNER], BATCH_ID, completeBatch.getBatchDataList(), failPostCallback);
jobSemaphore.acquire(2);
// Read batch data
BatchSpecificationMessage batchSpecificationMessage =
BatchSpecificationMessage.newBuilder()
.setSignerId(signerIDs[SIGNER])
.setBatchId(BATCH_ID)
.setStartPosition(0)
.build();
readBatchCallback = new ReadBatchCallback(completeBatch);
bulletinBoardClient.readBatch(batchSpecificationMessage, readBatchCallback);
jobSemaphore.acquire();
}
/**
* Posts a complete batch message
* Checks reading od the message
* @throws CommunicationException, SignatureException, InterruptedException
*/
@Test
public void testCompleteBatchPost() throws CommunicationException, SignatureException, InterruptedException {
final int SIGNER = 0;
final int BATCH_ID = 101;
final int BATCH_LENGTH = 50;
// Post batch
CompleteBatch completeBatch = createRandomBatch(SIGNER, BATCH_ID, BATCH_LENGTH);
bulletinBoardClient.postBatch(completeBatch,postCallback);
jobSemaphore.acquire();
// Read batch
BatchSpecificationMessage batchSpecificationMessage =
BatchSpecificationMessage.newBuilder()
.setSignerId(signerIDs[SIGNER])
.setBatchId(BATCH_ID)
.setStartPosition(0)
.build();
readBatchCallback = new ReadBatchCallback(completeBatch);
bulletinBoardClient.readBatch(batchSpecificationMessage, readBatchCallback);
jobSemaphore.acquire();
}
/**
* Tests that an unopened batch cannot be closed
* @throws CommunicationException, InterruptedException
*/
@Test
public void testInvalidBatchClose() throws CommunicationException, InterruptedException {
final int NON_EXISTENT_BATCH_ID = 999;
CloseBatchMessage closeBatchMessage =
CloseBatchMessage.newBuilder()
.setBatchId(NON_EXISTENT_BATCH_ID)
.setBatchLength(1)
.setSig(Crypto.Signature.getDefaultInstance())
.setTimestamp(Timestamp.newBuilder()
.setSeconds(9)
.setNanos(12)
.build())
.build();
// Try to close the (unopened) batch;
bulletinBoardClient.closeBatch(closeBatchMessage, failPostCallback);
jobSemaphore.acquire();
}
}

25
bulletin-board-server/build.gradle
View File

@ -48,7 +48,7 @@ dependencies {
// JDBC connections
compile 'org.springframework:spring-jdbc:4.2.+'
compile 'org.xerial:sqlite-jdbc:3.7.+'
compile 'org.xerial:sqlite-jdbc:3.8.+'
compile 'mysql:mysql-connector-java:5.1.+'
compile 'com.h2database:h2:1.0.+'
@ -79,9 +79,30 @@ test {
exclude '**/*IntegrationTest*'
}
task myTest(type: Test) {
include '**/*MySQL*Test*'
outputs.upToDateWhen { false }
}
task h2Test(type: Test) {
include '**/*H2*Test*'
outputs.upToDateWhen { false }
}
task liteTest(type: Test) {
include '**/*SQLite*Test*'
outputs.upToDateWhen { false }
}
task dbTest(type: Test) {
include '**/*H2*Test*'
include '**/*MySql*Test'
include '**/*MySQL*Test*'
include '**/*SQLite*Test*'
outputs.upToDateWhen { false }
}
task manualIntegration(type: Test) {
include '**/*IntegrationTest*'
}
task integrationTest(type: Test) {

694
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java
View File

@ -3,26 +3,36 @@ package meerkat.bulletinboard.sqlserver;
import java.sql.*;
import java.util.*;
import com.google.protobuf.ProtocolStringList;
import com.google.protobuf.*;
import meerkat.bulletinboard.*;
import meerkat.bulletinboard.sqlserver.mappers.*;
import static meerkat.bulletinboard.BulletinBoardConstants.*;
import meerkat.bulletinboard.BulletinBoardServer;
import meerkat.bulletinboard.sqlserver.mappers.EntryNumMapper;
import meerkat.bulletinboard.sqlserver.mappers.MessageMapper;
import meerkat.bulletinboard.sqlserver.mappers.SignatureMapper;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageOutputStream;
import meerkat.crypto.concrete.ECDSASignature;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto.Signature;
import meerkat.protobuf.Crypto.SignatureVerificationKey;
import meerkat.crypto.Digest;
import meerkat.crypto.concrete.SHA256Digest;
import static meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider.*;
import javax.sql.DataSource;
import meerkat.util.BulletinBoardUtils;
import meerkat.util.TimestampComparator;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.jdbc.support.GeneratedKeyHolder;
import org.springframework.jdbc.support.KeyHolder;
/**
* This is a generic SQL implementation of the BulletinBoardServer API.
*/
@ -40,24 +50,120 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
*/
public static enum QueryType {
FIND_MSG_ID(new String[] {"MsgId"}),
INSERT_MSG(new String[] {"MsgId","Msg"}),
INSERT_NEW_TAG(new String[] {"Tag"}),
CONNECT_TAG(new String[] {"EntryNum","Tag"}),
ADD_SIGNATURE(new String[] {"EntryNum","SignerId","Signature"}),
GET_SIGNATURES(new String[] {"EntryNum"}),
GET_MESSAGES(new String[] {});
FIND_MSG_ID(
new String[] {"MsgId"},
new int[] {Types.BLOB}
),
FIND_TAG_ID(
new String[] {"Tag"},
new int[] {Types.VARCHAR}
),
INSERT_MSG(
new String[] {"MsgId","TimeStamp","Msg"},
new int[] {Types.BLOB, Types.TIMESTAMP, Types.BLOB}
),
INSERT_NEW_TAG(
new String[] {"Tag"},
new int[] {Types.VARCHAR}
),
CONNECT_TAG(
new String[] {"EntryNum","Tag"},
new int[] {Types.INTEGER, Types.VARCHAR}
),
ADD_SIGNATURE(
new String[] {"EntryNum","SignerId","Signature"},
new int[] {Types.INTEGER, Types.BLOB, Types.BLOB}
),
GET_SIGNATURES(
new String[] {"EntryNum"},
new int[] {Types.INTEGER}
),
GET_MESSAGES(
new String[] {},
new int[] {}
),
COUNT_MESSAGES(
new String[] {},
new int[] {}
),
GET_MESSAGE_STUBS(
new String[] {},
new int[] {}
),
GET_LAST_MESSAGE_ENTRY(
new String[] {},
new int[] {}
),
GET_BATCH_MESSAGE_ENTRY(
new String[] {"SignerId", "BatchId"},
new int[] {Types.BLOB, Types.INTEGER}
),
CHECK_BATCH_LENGTH(
new String[] {"SignerId", "BatchId"},
new int[] {Types.BLOB, Types.INTEGER}
),
GET_BATCH_MESSAGE_DATA(
new String[] {"SignerId", "BatchId", "StartPosition"},
new int[] {Types.BLOB, Types.INTEGER, Types.INTEGER}
),
INSERT_BATCH_DATA(
new String[] {"SignerId", "BatchId", "SerialNum", "Data"},
new int[] {Types.BLOB, Types.INTEGER, Types.INTEGER, Types.BLOB}
),
CONNECT_BATCH_TAG(
new String[] {"SignerId", "BatchId", "Tag"},
new int[] {Types.BLOB, Types.INTEGER, Types.VARCHAR}
),
GET_BATCH_TAGS(
new String[] {"SignerId", "BatchId"},
new int[] {Types.BLOB, Types.INTEGER}
),
REMOVE_BATCH_TAGS(
new String[] {"SignerId", "BatchId"},
new int[] {Types.BLOB, Types.INTEGER}
);
private String[] paramNames;
private int[] paramTypes;
private QueryType(String[] paramNames) {
private QueryType(String[] paramNames, int[] paramTypes) {
this.paramNames = paramNames;
this.paramTypes = paramTypes;
}
public String[] getParamNames() {
return paramNames;
}
public String getParamName(int num) {
return paramNames[num];
}
public int[] getParamTypes() {
return paramTypes;
}
public int getParamType(int num) {
return paramTypes[num];
}
}
/**
@ -69,7 +175,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
MSG_ID("MsgId", Types.BLOB),
SIGNER_ID("SignerId", Types.BLOB),
TAG("Tag", Types.VARCHAR),
LIMIT("Limit", Types.INTEGER);
LIMIT("Limit", Types.INTEGER),
TIMESTAMP("TimeStamp", Types.TIMESTAMP);
private FilterTypeParam(String paramName, int paramType) {
this.paramName = paramName;
@ -85,8 +192,9 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
case MSG_ID:
return MSG_ID;
case EXACT_ENTRY: // Go through
case MAX_ENTRY:
case EXACT_ENTRY: // Go through
case MAX_ENTRY: // Go through
case MIN_ENTRY:
return ENTRY_NUM;
case SIGNER_ID:
@ -98,6 +206,10 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
case MAX_MESSAGES:
return LIMIT;
case BEFORE_TIME: // Go through
case AFTER_TIME:
return TIMESTAMP;
default:
return null;
}
@ -152,16 +264,22 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
}
/**
* This method returns the value of the parameter specified in a message filter
* @param messageFilter is the filter
* @return the object parameter for the SQL query embedded in the filter (this depends on the filter type)
*/
private Object getParam(MessageFilter messageFilter) {
switch (messageFilter.getType()) {
case MSG_ID: // Go through
case MSG_ID: // Go through
case SIGNER_ID:
return messageFilter.getId().toByteArray();
case EXACT_ENTRY: // Go through
case MAX_ENTRY:
case EXACT_ENTRY: // Go through
case MAX_ENTRY: // Go through
case MIN_ENTRY:
return messageFilter.getEntry();
case TAG:
@ -170,7 +288,11 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
case MAX_MESSAGES:
return messageFilter.getMaxMessages();
default:
case BEFORE_TIME: // Go through
case AFTER_TIME:
return BulletinBoardUtils.toSQLTimestamp(messageFilter.getTimestamp());
default: // Unsupported filter type
return null;
}
@ -193,7 +315,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
protected NamedParameterJdbcTemplate jdbcTemplate;
protected Digest digest;
protected BatchDigest digest;
protected BatchDigitalSignature signer;
protected List<SignatureVerificationKey> trusteeSignatureVerificationArray;
protected int minTrusteeSignatures;
@ -216,8 +339,6 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
*/
private void createSchema() throws SQLException {
final int TIMEOUT = 20;
for (String command : sqlQueryProvider.getSchemaCreationCommands()) {
jdbcTemplate.update(command,(Map) null);
}
@ -231,7 +352,8 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
public void init(String meerkatDB) throws CommunicationException {
// TODO write signature reading part.
digest = new SHA256Digest();
digest = new GenericBatchDigest(new SHA256Digest());
signer = new GenericBatchDigitalSignature(new ECDSASignature());
jdbcTemplate = new NamedParameterJdbcTemplate(sqlQueryProvider.getDataSource());
@ -264,12 +386,12 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
String sql;
sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.INSERT_NEW_TAG);
sql = sqlQueryProvider.getSQLString(QueryType.INSERT_NEW_TAG);
Map namedParameters[] = new HashMap[tags.length];
for (int i = 0 ; i < tags.length ; i++){
namedParameters[i] = new HashMap();
namedParameters[i].put("Tag", tags[i]);
namedParameters[i].put(QueryType.INSERT_NEW_TAG.getParamName(0), tags[i]);
}
jdbcTemplate.batchUpdate(sql, namedParameters);
@ -287,10 +409,17 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
.build();
}
@Override
public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException {
if (!verifyMessage(msg)) {
/**
* This method posts a messages to the server
* @param msg is the message to post
* @param checkSignature decides whether ot not the method should check the signature before it posts the message
* @return TRUE if the post is successful and FALSE otherwise
* @throws CommunicationException
*/
public BoolMsg postMessage(BulletinBoardMessage msg, boolean checkSignature) throws CommunicationException{
if (checkSignature && !verifyMessage(msg)) {
return boolToBoolMsg(false);
}
@ -299,28 +428,28 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
byte[] msgID;
long entryNum;
ProtocolStringList tagList;
String[] tags;
List<Signature> signatureList;
Signature[] signatures;
// Calculate message ID (depending only on the the unsigned message)
digest.reset();
digest.update(msg.getMsg());
msgID = digest.digest();
// Add message to table if needed and store entry number of message.
sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.FIND_MSG_ID);
sql = sqlQueryProvider.getSQLString(QueryType.FIND_MSG_ID);
Map namedParameters = new HashMap();
namedParameters.put("MsgId",msgID);
List<Long> entryNums = jdbcTemplate.query(sql, new MapSqlParameterSource(namedParameters), new EntryNumMapper());
namedParameters.put(QueryType.FIND_MSG_ID.getParamName(0),msgID);
List<Long> entryNums = jdbcTemplate.query(sql, namedParameters, new LongMapper());
if (entryNums.size() > 0){
@ -328,99 +457,106 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
} else{
sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.INSERT_MSG);
namedParameters.put("Msg", msg.getMsg().toByteArray());
sql = sqlQueryProvider.getSQLString(QueryType.INSERT_MSG);
namedParameters.put(QueryType.INSERT_MSG.getParamName(1), BulletinBoardUtils.toSQLTimestamp(msg.getMsg().getTimestamp()));
namedParameters.put(QueryType.INSERT_MSG.getParamName(2), msg.getMsg().toByteArray());
KeyHolder keyHolder = new GeneratedKeyHolder();
jdbcTemplate.update(sql,new MapSqlParameterSource(namedParameters),keyHolder);
jdbcTemplate.update(sql, new MapSqlParameterSource(namedParameters), keyHolder);
entryNum = keyHolder.getKey().longValue();
}
// Retrieve tags and store new ones in tag table.
try {
tagList = msg.getMsg().getTagList();
tags = new String[tagList.size()];
tags = tagList.toArray(tags);
insertNewTags(tags);
tagList = msg.getMsg().getTagList();
tags = new String[tagList.size()];
tags = tagList.toArray(tags);
insertNewTags(tags);
} catch (SQLException e) {
throw new CommunicationException(e.getMessage());
}
// Connect message to tags.
sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.CONNECT_TAG);
sql = sqlQueryProvider.getSQLString(QueryType.CONNECT_TAG);
namedParameterArray = new HashMap[tags.length];
for (int i = 0 ; i < tags.length ; i++) {
namedParameterArray[i] = new HashMap();
namedParameterArray[i].put("EntryNum", entryNum);
namedParameterArray[i].put("Tag", tags[i]);
namedParameterArray[i].put(QueryType.CONNECT_TAG.getParamName(0), entryNum);
namedParameterArray[i].put(QueryType.CONNECT_TAG.getParamName(1), tags[i]);
}
jdbcTemplate.batchUpdate(sql, namedParameterArray);
// Retrieve signatures.
signatureList = msg.getSigList();
signatures = new Signature[signatureList.size()];
signatures = signatureList.toArray(signatures);
signatureList = msg.getSigList();
signatures = new Signature[signatureList.size()];
signatures = signatureList.toArray(signatures);
// Connect message to signatures.
sql = sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.ADD_SIGNATURE);
sql = sqlQueryProvider.getSQLString(QueryType.ADD_SIGNATURE);
namedParameterArray = new HashMap[signatures.length];
for (int i = 0 ; i < signatures.length ; i++) {
namedParameterArray[i] = new HashMap();
namedParameterArray[i].put("EntryNum", entryNum);
namedParameterArray[i].put("SignerId", signatures[i].getSignerId().toByteArray());
namedParameterArray[i].put("Signature", signatures[i].toByteArray());
namedParameterArray[i].put(QueryType.ADD_SIGNATURE.getParamName(0), entryNum);
namedParameterArray[i].put(QueryType.ADD_SIGNATURE.getParamName(1), signatures[i].getSignerId().toByteArray());
namedParameterArray[i].put(QueryType.ADD_SIGNATURE.getParamName(2), signatures[i].toByteArray());
}
jdbcTemplate.batchUpdate(sql,namedParameterArray);
return boolToBoolMsg(true);
}
@Override
public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException {
public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException {
return postMessage(msg, true); // Perform a post and check the signature for authenticity
}
BulletinBoardMessageList.Builder resultListBuilder = BulletinBoardMessageList.newBuilder();
// SQL length is roughly 50 characters per filter + 50 for the query itself
StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1));
/**
* This is a container class for and SQL string builder and a MapSqlParameterSource to be used with it
*/
class SQLAndParameters {
MapSqlParameterSource namedParameters;
int paramNum;
public StringBuilder sql;
public MapSqlParameterSource parameters;
MessageMapper messageMapper = new MessageMapper();
SignatureMapper signatureMapper = new SignatureMapper();
public SQLAndParameters(int numOfFilters) {
sql = new StringBuilder(50 * numOfFilters);
parameters = new MapSqlParameterSource();
}
}
SQLAndParameters getSQLFromFilters(MessageFilterList filterList) {
SQLAndParameters result = new SQLAndParameters(filterList.getFilterCount());
List<MessageFilter> filters = new ArrayList<MessageFilter>(filterList.getFilterList());
boolean isFirstFilter = true;
Collections.sort(filters, new FilterTypeComparator());
// Check if Tag/Signature tables are required for filtering purposes
sqlBuilder.append(sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.GET_MESSAGES));
// Add conditions
namedParameters = new MapSqlParameterSource();
boolean isFirstFilter = true;
if (!filters.isEmpty()) {
sqlBuilder.append(" WHERE ");
result.sql.append(" WHERE ");
for (paramNum = 0 ; paramNum < filters.size() ; paramNum++) {
for (int paramNum = 0 ; paramNum < filters.size() ; paramNum++) {
MessageFilter filter = filters.get(paramNum);
@ -428,15 +564,15 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
if (isFirstFilter) {
isFirstFilter = false;
} else {
sqlBuilder.append(" AND ");
result.sql.append(" AND ");
}
}
sqlBuilder.append(sqlQueryProvider.getCondition(filter.getType(), paramNum));
result.sql.append(sqlQueryProvider.getCondition(filter.getType(), paramNum));
SQLQueryProvider.FilterTypeParam filterTypeParam = SQLQueryProvider.FilterTypeParam.getFilterTypeParamName(filter.getType());
FilterTypeParam filterTypeParam = FilterTypeParam.getFilterTypeParamName(filter.getType());
namedParameters.addValue(
result.parameters.addValue(
filterTypeParam.getParamName() + Integer.toString(paramNum),
getParam(filter),
filterTypeParam.getParamType(),
@ -446,35 +582,381 @@ public class BulletinBoardSQLServer implements BulletinBoardServer{
}
return result;
}
/**
* Used to retrieve just basic information about messages to allow calculation of checksum
* @param filterList is a filter list that defines which messages the client is interested in
* @return a list of Bulletin Board Messages that contain just the entry number, timestamp and message ID for each message
* The message ID is returned inside the message data field
*/
protected List<BulletinBoardMessage> readMessageStubs(MessageFilterList filterList) {
StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1));
sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.GET_MESSAGE_STUBS));
// Get Conditions
SQLAndParameters sqlAndParameters = getSQLFromFilters(filterList);
sqlBuilder.append(sqlAndParameters.sql);
// Run query
List<BulletinBoardMessage.Builder> msgBuilders = jdbcTemplate.query(sqlBuilder.toString(), namedParameters, messageMapper);
return jdbcTemplate.query(sqlBuilder.toString(), sqlAndParameters.parameters, new MessageStubMapper());
// Compile list of messages
}
for (BulletinBoardMessage.Builder msgBuilder : msgBuilders) {
// Retrieve signatures
@Override
public void readMessages(MessageFilterList filterList, MessageOutputStream<BulletinBoardMessage> out) throws CommunicationException {
namedParameters = new MapSqlParameterSource();
namedParameters.addValue("EntryNum", msgBuilder.getEntryNum());
BulletinBoardMessageList.Builder resultListBuilder = BulletinBoardMessageList.newBuilder();
List<Signature> signatures = jdbcTemplate.query(
sqlQueryProvider.getSQLString(SQLQueryProvider.QueryType.GET_SIGNATURES),
namedParameters,
signatureMapper);
// SQL length is roughly 50 characters per filter + 50 for the query itself
StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1));
// Append signatures
msgBuilder.addAllSig(signatures);
// Check if Tag/Signature tables are required for filtering purposes
// Finalize message and add to message list.
sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.GET_MESSAGES));
resultListBuilder.addMessage(msgBuilder.build());
// Get conditions
SQLAndParameters sqlAndParameters = getSQLFromFilters(filterList);
sqlBuilder.append(sqlAndParameters.sql);
// Run query and stream the output using a MessageCallbackHandler
jdbcTemplate.query(sqlBuilder.toString(), sqlAndParameters.parameters, new MessageCallbackHandler(jdbcTemplate, sqlQueryProvider, out));
}
/**
* This method returns a string representation of the tag associated with a batch ID
* @param batchId is the given batch ID
* @return the String representation of the tag
*/
private String batchIdToTag(int batchId) {
return BATCH_ID_TAG_PREFIX + Integer.toString(batchId);
}
/**
* This method checks if a specified batch exists and is already closed
* @param signerId is the ID of the publisher of the batch
* @param batchId is the unique (per signer) batch ID
* @return TRUE if the batch is closed and FALSE if it is still open or doesn't exist at all
*/
private boolean isBatchClosed(ByteString signerId, int batchId) throws CommunicationException {
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.SIGNER_ID)
.setId(signerId)
.build())
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag(BATCH_TAG)
.build())
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag(batchIdToTag(batchId))
.build())
.build();
// SQL length is roughly 50 characters per filter + 50 for the query itself
StringBuilder sqlBuilder = new StringBuilder(50 * (filterList.getFilterCount() + 1));
// Check if Tag/Signature tables are required for filtering purposes
sqlBuilder.append(sqlQueryProvider.getSQLString(QueryType.COUNT_MESSAGES));
// Get conditions
SQLAndParameters sqlAndParameters = getSQLFromFilters(filterList);
sqlBuilder.append(sqlAndParameters.sql);
// Run query and stream the output using a MessageCallbackHandler
List<Long> count = jdbcTemplate.query(sqlBuilder.toString(), sqlAndParameters.parameters, new LongMapper());
return (count.size() > 0) && (count.get(0) > 0);
}
@Override
public BoolMsg beginBatch(BeginBatchMessage message) throws CommunicationException {
// Check if batch is closed
if (isBatchClosed(message.getSignerId(), message.getBatchId())) {
return BoolMsg.newBuilder().setValue(false).build();
}
// Add new tags to table
ProtocolStringList tagList = message.getTagList();
String[] tags = new String[tagList.size()];
tags = tagList.toArray(tags);
try {
insertNewTags(tags);
} catch (SQLException e) {
throw new CommunicationException(e.getMessage());
}
// Connect tags
String sql = sqlQueryProvider.getSQLString(QueryType.CONNECT_BATCH_TAG);
MapSqlParameterSource namedParameters[] = new MapSqlParameterSource[tags.length];
for (int i=0 ; i < tags.length ; i++) {
namedParameters[i] = new MapSqlParameterSource();
namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(0),message.getSignerId().toByteArray());
namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(1),message.getBatchId());
namedParameters[i].addValue(QueryType.CONNECT_BATCH_TAG.getParamName(2),tags[i]);
}
jdbcTemplate.batchUpdate(sql,namedParameters);
return BoolMsg.newBuilder().setValue(true).build();
}
@Override
public BoolMsg postBatchMessage(BatchMessage batchMessage) throws CommunicationException{
// Check if batch is closed
if (isBatchClosed(batchMessage.getSignerId(), batchMessage.getBatchId())) {
return BoolMsg.newBuilder().setValue(false).build();
}
// Add data
String sql = sqlQueryProvider.getSQLString(QueryType.INSERT_BATCH_DATA);
MapSqlParameterSource namedParameters = new MapSqlParameterSource();
namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(0),batchMessage.getSignerId().toByteArray());
namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(1),batchMessage.getBatchId());
namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(2),batchMessage.getSerialNum());
namedParameters.addValue(QueryType.INSERT_BATCH_DATA.getParamName(3),batchMessage.getData().toByteArray());
jdbcTemplate.update(sql, namedParameters);
return BoolMsg.newBuilder().setValue(true).build();
}
@Override
public BoolMsg closeBatchMessage(CloseBatchMessage message) throws CommunicationException {
ByteString signerId = message.getSig().getSignerId();
int batchId = message.getBatchId();
KeyHolder keyHolder = new GeneratedKeyHolder();
// Check batch size
String sql = sqlQueryProvider.getSQLString(QueryType.CHECK_BATCH_LENGTH);
MapSqlParameterSource namedParameters = new MapSqlParameterSource();
namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(0),signerId.toByteArray());
namedParameters.addValue(QueryType.CHECK_BATCH_LENGTH.getParamName(1),batchId);
List<Long> lengthResult = jdbcTemplate.query(sql, namedParameters, new LongMapper());
if (lengthResult.get(0) != message.getBatchLength()) {
return BoolMsg.newBuilder().setValue(false).build();
}
// Get Tags and add them to CompleteBatch
sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_TAGS);
namedParameters = new MapSqlParameterSource();
namedParameters.addValue(QueryType.GET_BATCH_TAGS.getParamName(0),signerId);
namedParameters.addValue(QueryType.GET_BATCH_TAGS.getParamName(1),batchId);
List<String> tags = jdbcTemplate.query(sql, namedParameters, new StringMapper());
CompleteBatch completeBatch = new CompleteBatch(
BeginBatchMessage.newBuilder()
.setSignerId(signerId)
.setBatchId(batchId)
.addAllTag(tags)
.build()
);
// Add timestamp to CompleteBatch
completeBatch.setTimestamp(message.getTimestamp());
// Add actual batch data to CompleteBatch
sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA);
namedParameters = new MapSqlParameterSource();
namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),signerId.toByteArray());
namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),batchId);
namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),0); // Read from the beginning
completeBatch.appendBatchData(jdbcTemplate.query(sql, namedParameters, new BatchDataMapper()));
// Verify signature
completeBatch.setSignature(message.getSig());
// try {
// TODO: Actual verification
// //signer.verify(completeBatch);
// } catch (CertificateException | InvalidKeyException | SignatureException e) {
// return BoolMsg.newBuilder().setValue(false).build();
// }
// Batch verified: finalize it
// Calculate message ID
digest.reset();
digest.update(completeBatch);
MessageID msgID = MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build();
// Create Bulletin Board message
BulletinBoardMessage bulletinBoardMessage = BulletinBoardMessage.newBuilder()
.addSig(message.getSig())
.setMsg(UnsignedBulletinBoardMessage.newBuilder()
.addAllTag(tags)
.addTag(BATCH_TAG)
.addTag(batchIdToTag(batchId))
.setData(message.getSig().getSignerId())
.setTimestamp(message.getTimestamp())
.build())
.build();
// Post message without checking signature validity
postMessage(bulletinBoardMessage, false);
// Remove tags from temporary table
sql = sqlQueryProvider.getSQLString(QueryType.REMOVE_BATCH_TAGS);
namedParameters = new MapSqlParameterSource();
namedParameters.addValue(QueryType.REMOVE_BATCH_TAGS.getParamName(0), signerId.toByteArray());
namedParameters.addValue(QueryType.REMOVE_BATCH_TAGS.getParamName(1), batchId);
jdbcTemplate.update(sql, namedParameters);
// Return TRUE
return BoolMsg.newBuilder().setValue(true).build();
}
@Override
public void readBatch(BatchSpecificationMessage message, MessageOutputStream<BatchData> out) throws CommunicationException, IllegalArgumentException{
// Check that batch is closed
if (!isBatchClosed(message.getSignerId(), message.getBatchId())) {
throw new IllegalArgumentException("No such batch");
}
String sql = sqlQueryProvider.getSQLString(QueryType.GET_BATCH_MESSAGE_DATA);
MapSqlParameterSource namedParameters = new MapSqlParameterSource();
namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),message.getSignerId().toByteArray());
namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),message.getBatchId());
namedParameters.addValue(QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2),message.getStartPosition());
jdbcTemplate.query(sql, namedParameters, new BatchDataCallbackHandler(out));
}
/**
* Finds the entry number of the last entry in the database
* @return the entry number, or -1 if no entries are found
*/
protected long getLastMessageEntry() {
String sql = sqlQueryProvider.getSQLString(QueryType.GET_LAST_MESSAGE_ENTRY);
List<Long> resultList = jdbcTemplate.query(sql, new LongMapper());
if (resultList.size() <= 0){
return -1;
}
return resultList.get(0);
}
/**
* Searches for the latest time of sync of the DB relative to a given query and returns the metadata needed to complete the sync
* The checksum up to (and including) each given timestamp is calculated using bitwise XOR on 8-byte sized blocks of the message IDs
* @param syncQuery contains a succinct representation of states to compare to
* @return the current last entry num and latest time of sync if there is one; -1 as last entry and empty timestamp otherwise
* @throws CommunicationException
*/
@Override
public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException {
if (syncQuery == null){
return SyncQueryResponse.newBuilder()
.setLastEntryNum(-1)
.setLastTimeOfSync(com.google.protobuf.Timestamp.getDefaultInstance())
.build();
}
com.google.protobuf.Timestamp lastTimeOfSync = null;
TimestampComparator timestampComparator = new TimestampComparator();
long lastEntryNum = getLastMessageEntry();
Iterator<SingleSyncQuery> queryIterator = syncQuery.getQueryList().iterator();
SingleSyncQuery currentQuery = queryIterator.next();
List<BulletinBoardMessage> messageStubs = readMessageStubs(syncQuery.getFilterList());
Checksum checksum = new SimpleChecksum();
for (BulletinBoardMessage message : messageStubs){
// Check for end of current query
if (timestampComparator.compare(message.getMsg().getTimestamp(), currentQuery.getTimeOfSync()) > 0){
if (checksum.getChecksum() == currentQuery.getChecksum()){
lastTimeOfSync = currentQuery.getTimeOfSync();
} else {
break;
}
if (queryIterator.hasNext()){
currentQuery = queryIterator.next();
} else{
break;
}
}
// Advance checksum
ByteString messageID = message.getMsg().getData(); // The data field contains the message ID
checksum.update(messageID);
}
//Combine results and return.
return resultListBuilder.build();
if (checksum.getChecksum() == currentQuery.getChecksum()){
lastTimeOfSync = currentQuery.getTimeOfSync();
}
if (lastTimeOfSync == null){
return SyncQueryResponse.newBuilder()
.setLastEntryNum(-1)
.setLastTimeOfSync(com.google.protobuf.Timestamp.getDefaultInstance())
.build();
} else{
return SyncQueryResponse.newBuilder()
.setLastEntryNum(lastEntryNum)
.setLastTimeOfSync(lastTimeOfSync)
.build();
}
}

97
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java
View File

@ -7,6 +7,7 @@ import javax.naming.InitialContext;
import javax.naming.NamingException;
import javax.sql.DataSource;
import java.text.MessageFormat;
import java.util.LinkedList;
import java.util.List;
@ -42,9 +43,20 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
case FIND_MSG_ID:
return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId";
case FIND_TAG_ID:
return MessageFormat.format(
"SELECT TagId FROM TagTable WHERE Tag = :{0}",
QueryType.FIND_TAG_ID.getParamName(0));
case GET_MESSAGES:
return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable";
case COUNT_MESSAGES:
return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable";
case GET_MESSAGE_STUBS:
return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable";
case GET_SIGNATURES:
return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum";
@ -55,6 +67,66 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
return "INSERT INTO TagTable(Tag) SELECT DISTINCT :Tag AS NewTag FROM UtilityTable WHERE"
+ " NOT EXISTS (SELECT 1 FROM TagTable AS SubTable WHERE SubTable.Tag = :Tag)";
case GET_LAST_MESSAGE_ENTRY:
return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable";
case GET_BATCH_MESSAGE_ENTRY:
return MessageFormat.format(
"SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"
+ " INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum"
+ " INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum"
+ " INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId"
+ " WHERE SignatureTable.SignerId = :{0}"
+ " AND TagTable.Tag = :{1}",
QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0),
QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1));
case GET_BATCH_MESSAGE_DATA:
return MessageFormat.format(
"SELECT Data FROM BatchTable"
+ " WHERE SignerId = :{0} AND BatchId = :{1} AND SerialNum >= :{2}"
+ " ORDER BY SerialNum ASC",
QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),
QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),
QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2));
case INSERT_BATCH_DATA:
return MessageFormat.format(
"INSERT INTO BatchTable (SignerId, BatchId, SerialNum, Data)"
+ " VALUES (:{0}, :{1}, :{2}, :{3})",
QueryType.INSERT_BATCH_DATA.getParamName(0),
QueryType.INSERT_BATCH_DATA.getParamName(1),
QueryType.INSERT_BATCH_DATA.getParamName(2),
QueryType.INSERT_BATCH_DATA.getParamName(3));
case CHECK_BATCH_LENGTH:
return MessageFormat.format(
"SELECT COUNT(Data) AS BatchLength FROM BatchTable"
+ " WHERE SignerId = :{0} AND BatchId = :{1}",
QueryType.CHECK_BATCH_LENGTH.getParamName(0),
QueryType.CHECK_BATCH_LENGTH.getParamName(1));
case CONNECT_BATCH_TAG:
return MessageFormat.format(
"INSERT INTO BatchTagTable (SignerId, BatchId, TagId) SELECT :{0}, :{1}, TagId FROM TagTable"
+ " WHERE Tag = :{2}",
QueryType.CONNECT_BATCH_TAG.getParamName(0),
QueryType.CONNECT_BATCH_TAG.getParamName(1),
QueryType.CONNECT_BATCH_TAG.getParamName(2));
case GET_BATCH_TAGS:
return MessageFormat.format(
"SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.TagId"
+ " WHERE SignerId = :{0} AND BatchId = :{1} ORDER BY Tag ASC",
QueryType.GET_BATCH_TAGS.getParamName(0),
QueryType.GET_BATCH_TAGS.getParamName(1));
case REMOVE_BATCH_TAGS:
return MessageFormat.format(
"DELETE FROM BatchTagTable WHERE SignerId = :{0} AND BatchId = :{1}",
QueryType.REMOVE_BATCH_TAGS.getParamName(0),
QueryType.REMOVE_BATCH_TAGS.getParamName(1));
default:
throw new IllegalArgumentException("Cannot serve a query of type " + queryType);
}
@ -71,10 +143,12 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
return "MsgTable.EntryNum = :EntryNum" + serialString;
case MAX_ENTRY:
return "MsgTable.EntryNum <= :EntryNum" + serialString;
case MIN_ENTRY:
return "MsgTable.EntryNum >= :EntryNum" + serialString;
case MAX_MESSAGES:
return "LIMIT :Limit" + serialString;
case MSG_ID:
return "MsgTable.MsgId = MsgId" + serialString;
return "MsgTable.MsgId = :MsgId" + serialString;
case SIGNER_ID:
return "EXISTS (SELECT 1 FROM SignatureTable"
+ " WHERE SignatureTable.SignerId = :SignerId" + serialString + " AND SignatureTable.EntryNum = MsgTable.EntryNum)";
@ -82,6 +156,13 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
return "EXISTS (SELECT 1 FROM TagTable"
+ " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId"
+ " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)";
case BEFORE_TIME:
return "MsgTable.ExactTime <= :TimeStamp";
case AFTER_TIME:
return "MsgTable.ExactTime >= :TimeStamp";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
@ -94,6 +175,7 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
switch(filterType) {
case EXACT_ENTRY: // Go through
case MAX_ENTRY: // Go through
case MIN_ENTRY: // Go through
case MAX_MESSAGES:
return "INT";
@ -124,7 +206,7 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
public List<String> getSchemaCreationCommands() {
List<String> list = new LinkedList<String>();
list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB UNIQUE, Msg BLOB)");
list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB UNIQUE, ExactTime TIMESTAMP, Msg BLOB)");
list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INT NOT NULL AUTO_INCREMENT PRIMARY KEY, Tag VARCHAR(50) UNIQUE)");
@ -139,6 +221,14 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
list.add("CREATE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId)");
list.add("CREATE UNIQUE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId, EntryNum)");
list.add("CREATE TABLE IF NOT EXISTS BatchTable (SignerId TINYBLOB, BatchId INT, SerialNum INT, Data BLOB,"
+ " UNIQUE(SignerId, BatchId, SerialNum))");
list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (SignerId TINYBLOB, BatchId INT, TagId INT,"
+ " FOREIGN KEY (TagId) REFERENCES TagTable(TagId))");
list.add("CREATE INDEX IF NOT EXISTS BatchIndex ON BatchTagTable(SignerId, BatchId)");
// This is used to create a simple table with one entry.
// It is used for implementing a workaround for the missing INSERT IGNORE syntax
list.add("CREATE TABLE IF NOT EXISTS UtilityTable (Entry INT)");
@ -152,6 +242,9 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
List<String> list = new LinkedList<String>();
list.add("DROP TABLE IF EXISTS UtilityTable");
list.add("DROP INDEX IF EXISTS BatchIndex");
list.add("DROP TABLE IF EXISTS BatchTagTable");
list.add("DROP TABLE IF EXISTS BatchTable");
list.add("DROP INDEX IF EXISTS SignerIdIndex");
list.add("DROP TABLE IF EXISTS MsgTagTable");
list.add("DROP TABLE IF EXISTS SignatureTable");

141
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java
View File

@ -1,10 +1,12 @@
package meerkat.bulletinboard.sqlserver;
import com.mysql.jdbc.jdbc2.optional.MysqlDataSource;
import meerkat.bulletinboard.BulletinBoardConstants;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider;
import meerkat.protobuf.BulletinBoardAPI.FilterType;
import javax.sql.DataSource;
import java.text.MessageFormat;
import java.util.LinkedList;
import java.util.List;
@ -32,21 +34,117 @@ public class MySQLQueryProvider implements SQLQueryProvider {
public String getSQLString(QueryType queryType) throws IllegalArgumentException{
switch(queryType) {
case ADD_SIGNATURE:
return "INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:EntryNum, :SignerId, :Signature)";
return MessageFormat.format(
"INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:{0}, :{1}, :{2})",
QueryType.ADD_SIGNATURE.getParamName(0),
QueryType.ADD_SIGNATURE.getParamName(1),
QueryType.ADD_SIGNATURE.getParamName(2));
case CONNECT_TAG:
return "INSERT IGNORE INTO MsgTagTable (TagId, EntryNum)"
+ " SELECT TagTable.TagId, :EntryNum AS EntryNum FROM TagTable WHERE Tag = :Tag";
return MessageFormat.format(
"INSERT IGNORE INTO MsgTagTable (TagId, EntryNum)"
+ " SELECT TagTable.TagId, :{0} AS EntryNum FROM TagTable WHERE Tag = :{1}",
QueryType.CONNECT_TAG.getParamName(0),
QueryType.CONNECT_TAG.getParamName(1));
case FIND_MSG_ID:
return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId";
return MessageFormat.format(
"SELECT EntryNum From MsgTable WHERE MsgId = :{0}",
QueryType.FIND_MSG_ID.getParamName(0));
case FIND_TAG_ID:
return MessageFormat.format(
"SELECT TagId FROM TagTable WHERE Tag = :{0}",
QueryType.FIND_TAG_ID.getParamName(0));
case GET_MESSAGES:
return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable";
case COUNT_MESSAGES:
return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable";
case GET_MESSAGE_STUBS:
return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable";
case GET_SIGNATURES:
return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum";
return MessageFormat.format(
"SELECT Signature FROM SignatureTable WHERE EntryNum = :{0}",
QueryType.GET_SIGNATURES.getParamName(0));
case INSERT_MSG:
return "INSERT INTO MsgTable (MsgId, Msg) VALUES(:MsgId, :Msg)";
return MessageFormat.format(
"INSERT INTO MsgTable (MsgId, ExactTime, Msg) VALUES(:{0}, :{1}, :{2})",
QueryType.INSERT_MSG.getParamName(0),
QueryType.INSERT_MSG.getParamName(1),
QueryType.INSERT_MSG.getParamName(2));
case INSERT_NEW_TAG:
return "INSERT IGNORE INTO TagTable(Tag) VALUES (:Tag)";
return MessageFormat.format(
"INSERT IGNORE INTO TagTable(Tag) VALUES (:{0})",
QueryType.INSERT_NEW_TAG.getParamName(0));
case GET_LAST_MESSAGE_ENTRY:
return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable";
case GET_BATCH_MESSAGE_ENTRY:
return MessageFormat.format(
"SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"
+ " INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum"
+ " INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum"
+ " INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId"
+ " WHERE SignatureTable.SignerId = :{0}"
+ " AND TagTable.Tag = :{1}",
QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0),
QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1));
case GET_BATCH_MESSAGE_DATA:
return MessageFormat.format(
"SELECT Data FROM BatchTable"
+ " WHERE SignerId = :{0} AND BatchId = :{1} AND SerialNum >= :{2}"
+ " ORDER BY SerialNum ASC",
QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),
QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),
QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2));
case INSERT_BATCH_DATA:
return MessageFormat.format(
"INSERT INTO BatchTable (SignerId, BatchId, SerialNum, Data)"
+ " VALUES (:{0}, :{1}, :{2}, :{3})",
QueryType.INSERT_BATCH_DATA.getParamName(0),
QueryType.INSERT_BATCH_DATA.getParamName(1),
QueryType.INSERT_BATCH_DATA.getParamName(2),
QueryType.INSERT_BATCH_DATA.getParamName(3));
case CHECK_BATCH_LENGTH:
return MessageFormat.format(
"SELECT COUNT(Data) AS BatchLength FROM BatchTable"
+ " WHERE SignerId = :{0} AND BatchId = :{1}",
QueryType.CHECK_BATCH_LENGTH.getParamName(0),
QueryType.CHECK_BATCH_LENGTH.getParamName(1));
case CONNECT_BATCH_TAG:
return MessageFormat.format(
"INSERT INTO BatchTagTable (SignerId, BatchId, TagId) SELECT :{0}, :{1}, TagId FROM TagTable"
+ " WHERE Tag = :{2}",
QueryType.CONNECT_BATCH_TAG.getParamName(0),
QueryType.CONNECT_BATCH_TAG.getParamName(1),
QueryType.CONNECT_BATCH_TAG.getParamName(2));
case GET_BATCH_TAGS:
return MessageFormat.format(
"SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.TagId"
+ " WHERE SignerId = :{0} AND BatchId = :{1} ORDER BY Tag ASC",
QueryType.GET_BATCH_TAGS.getParamName(0),
QueryType.GET_BATCH_TAGS.getParamName(1));
case REMOVE_BATCH_TAGS:
return MessageFormat.format(
"DELETE FROM BatchTagTable WHERE SignerId = :{0} AND BatchId = :{1}",
QueryType.REMOVE_BATCH_TAGS.getParamName(0),
QueryType.REMOVE_BATCH_TAGS.getParamName(1));
default:
throw new IllegalArgumentException("Cannot serve a query of type " + queryType);
}
@ -63,6 +161,8 @@ public class MySQLQueryProvider implements SQLQueryProvider {
return "MsgTable.EntryNum = :EntryNum" + serialString;
case MAX_ENTRY:
return "MsgTable.EntryNum <= :EntryNum" + serialString;
case MIN_ENTRY:
return "MsgTable.EntryNum >= :EntryNum" + serialString;
case MAX_MESSAGES:
return "LIMIT :Limit" + serialString;
case MSG_ID:
@ -74,6 +174,13 @@ public class MySQLQueryProvider implements SQLQueryProvider {
return "EXISTS (SELECT 1 FROM TagTable"
+ " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId"
+ " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)";
case BEFORE_TIME:
return "MsgTable.ExactTime <= :TimeStamp";
case AFTER_TIME:
return "MsgTable.ExactTime >= :TimeStamp";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
@ -86,6 +193,7 @@ public class MySQLQueryProvider implements SQLQueryProvider {
switch(filterType) {
case EXACT_ENTRY: // Go through
case MAX_ENTRY: // Go through
case MIN_ENTRY: // Go through
case MAX_MESSAGES:
return "INT";
@ -96,6 +204,10 @@ public class MySQLQueryProvider implements SQLQueryProvider {
case TAG:
return "VARCHAR";
case AFTER_TIME: // Go through
case BEFORE_TIME:
return "TIMESTAMP";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
@ -111,6 +223,7 @@ public class MySQLQueryProvider implements SQLQueryProvider {
dataSource.setDatabaseName(dbName);
dataSource.setUser(username);
dataSource.setPassword(password);
dataSource.setAllowMultiQueries(true);
return dataSource;
}
@ -119,7 +232,8 @@ public class MySQLQueryProvider implements SQLQueryProvider {
public List<String> getSchemaCreationCommands() {
List<String> list = new LinkedList<String>();
list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB, Msg BLOB, UNIQUE(MsgId(50)))");
list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY,"
+ " MsgId TINYBLOB, ExactTime TIMESTAMP, Msg BLOB, UNIQUE(MsgId(50)))");
list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INT NOT NULL AUTO_INCREMENT PRIMARY KEY, Tag VARCHAR(50), UNIQUE(Tag))");
@ -129,7 +243,14 @@ public class MySQLQueryProvider implements SQLQueryProvider {
+ " CONSTRAINT UNIQUE (EntryNum, TagID))");
list.add("CREATE TABLE IF NOT EXISTS SignatureTable (EntryNum INT, SignerId TINYBLOB, Signature TINYBLOB,"
+ " INDEX(SignerId(32)), CONSTRAINT Uni UNIQUE(SignerId(32), EntryNum), CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))");
+ " INDEX(SignerId(32)), CONSTRAINT Unique_Signature UNIQUE(SignerId(32), EntryNum),"
+ " CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))");
list.add("CREATE TABLE IF NOT EXISTS BatchTable (SignerId TINYBLOB, BatchId INT, SerialNum INT, Data BLOB,"
+ " CONSTRAINT Unique_Batch UNIQUE(SignerId(32), BatchId, SerialNum))");
list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (SignerId TINYBLOB, BatchId INT, TagId INT,"
+ " INDEX(SignerId(32), BatchId), CONSTRAINT FOREIGN KEY (TagId) REFERENCES TagTable(TagId))");
return list;
}
@ -138,6 +259,8 @@ public class MySQLQueryProvider implements SQLQueryProvider {
public List<String> getSchemaDeletionCommands() {
List<String> list = new LinkedList<String>();
list.add("DROP TABLE IF EXISTS BatchTagTable");
list.add("DROP TABLE IF EXISTS BatchTable");
list.add("DROP TABLE IF EXISTS MsgTagTable");
list.add("DROP TABLE IF EXISTS SignatureTable");
list.add("DROP TABLE IF EXISTS TagTable");

120
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java
View File

@ -4,6 +4,7 @@ import meerkat.protobuf.BulletinBoardAPI.*;
import org.sqlite.SQLiteDataSource;
import javax.sql.DataSource;
import java.text.MessageFormat;
import java.util.LinkedList;
import java.util.List;
@ -25,19 +26,97 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi
switch(queryType) {
case ADD_SIGNATURE:
return "INSERT OR IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:EntryNum,:SignerId,:Signature)";
case CONNECT_TAG:
return "INSERT OR IGNORE INTO MsgTagTable (TagId, EntryNum)"
+ " SELECT TagTable.TagId, :EntryNum AS EntryNum FROM TagTable WHERE Tag = :Tag";
case FIND_MSG_ID:
return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId";
case FIND_TAG_ID:
return MessageFormat.format(
"SELECT TagId FROM TagTable WHERE Tag = :{0}",
QueryType.FIND_TAG_ID.getParamName(0));
case GET_MESSAGES:
return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable";
case COUNT_MESSAGES:
return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable";
case GET_MESSAGE_STUBS:
return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable";
case GET_SIGNATURES:
return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum";
case INSERT_MSG:
return "INSERT INTO MsgTable (MsgId, Msg) VALUES(:MsgId,:Msg)";
case INSERT_NEW_TAG:
return "INSERT OR IGNORE INTO TagTable(Tag) VALUES (:Tag)";
case GET_LAST_MESSAGE_ENTRY:
return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable";
case GET_BATCH_MESSAGE_ENTRY:
return MessageFormat.format(
"SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable"
+ " INNER JOIN SignatureTable ON MsgTable.EntryNum = SignatureTable.EntryNum"
+ " INNER JOIN MsgTagTable ON MsgTable.EntryNum = MsgTagTable.EntryNum"
+ " INNER JOIN TagTable ON MsgTagTable.TagId = TagTable.TagId"
+ " WHERE SignatureTable.SignerId = :{0}"
+ " AND TagTable.Tag = :{1}",
QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(0),
QueryType.GET_BATCH_MESSAGE_ENTRY.getParamName(1));
case GET_BATCH_MESSAGE_DATA:
return MessageFormat.format(
"SELECT Data FROM BatchTable"
+ " WHERE SignerId = :{0} AND BatchId = :{1} AND SerialNum >= :{2}"
+ " ORDER BY SerialNum ASC",
QueryType.GET_BATCH_MESSAGE_DATA.getParamName(0),
QueryType.GET_BATCH_MESSAGE_DATA.getParamName(1),
QueryType.GET_BATCH_MESSAGE_DATA.getParamName(2));
case INSERT_BATCH_DATA:
return MessageFormat.format(
"INSERT INTO BatchTable (SignerId, BatchId, SerialNum, Data)"
+ " VALUES (:{0}, :{1}, :{2}, :{3})",
QueryType.INSERT_BATCH_DATA.getParamName(0),
QueryType.INSERT_BATCH_DATA.getParamName(1),
QueryType.INSERT_BATCH_DATA.getParamName(2),
QueryType.INSERT_BATCH_DATA.getParamName(3));
case CHECK_BATCH_LENGTH:
return MessageFormat.format(
"SELECT COUNT(Data) AS BatchLength FROM BatchTable"
+ " WHERE SignerId = :{0} AND BatchId = :{1}",
QueryType.CHECK_BATCH_LENGTH.getParamName(0),
QueryType.CHECK_BATCH_LENGTH.getParamName(1));
case CONNECT_BATCH_TAG:
return MessageFormat.format(
"INSERT INTO BatchTagTable (SignerId, BatchId, TagId) SELECT :{0}, :{1}, TagId FROM TagTable"
+ " WHERE Tag = :{2}",
QueryType.CONNECT_BATCH_TAG.getParamName(0),
QueryType.CONNECT_BATCH_TAG.getParamName(1),
QueryType.CONNECT_BATCH_TAG.getParamName(2));
case GET_BATCH_TAGS:
return MessageFormat.format(
"SELECT Tag FROM TagTable INNER JOIN BatchTagTable ON TagTable.TagId = BatchTagTable.TagId"
+ " WHERE SignerId = :{0} AND BatchId = :{1} ORDER BY Tag ASC",
QueryType.GET_BATCH_TAGS.getParamName(0),
QueryType.GET_BATCH_TAGS.getParamName(1));
case REMOVE_BATCH_TAGS:
return MessageFormat.format(
"DELETE FROM BatchTagTable WHERE SignerId = :{0} AND BatchId = :{1}",
QueryType.REMOVE_BATCH_TAGS.getParamName(0),
QueryType.REMOVE_BATCH_TAGS.getParamName(1));
default:
throw new IllegalArgumentException("Cannot serve a query of type " + queryType);
}
@ -52,19 +131,34 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi
switch(filterType) {
case EXACT_ENTRY:
return "MsgTable.EntryNum = :EntryNum" + serialString;
case MAX_ENTRY:
return "MsgTable.EntryNum <= :EntryNum" + serialString;
case MIN_ENTRY:
return "MsgTable.EntryNum <= :EntryNum" + serialString;
case MAX_MESSAGES:
return "LIMIT = :Limit" + serialString;
case MSG_ID:
return "MsgTable.MsgId = :MsgId" + serialString;
case SIGNER_ID:
return "EXISTS (SELECT 1 FROM SignatureTable"
+ " WHERE SignatureTable.SignerId = :SignerId" + serialString + " AND SignatureTable.EntryNum = MsgTable.EntryNum)";
case TAG:
return "EXISTS (SELECT 1 FROM TagTable"
+ " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId"
+ " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)";
case BEFORE_TIME:
return "MsgTable.ExactTime <= :TimeStamp";
case AFTER_TIME:
return "MsgTable.ExactTime >= :TimeStamp";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
@ -73,15 +167,32 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi
@Override
public String getConditionParamTypeName(FilterType filterType) throws IllegalArgumentException {
return null; //TODO: write this.
switch(filterType) {
case EXACT_ENTRY: // Go through
case MAX_ENTRY: // Go through
case MIN_ENTRY: // Go through
case MAX_MESSAGES:
return "INTEGER";
case MSG_ID: // Go through
case SIGNER_ID:
return "BLOB";
case TAG:
return "VARCHAR";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
}
@Override
public DataSource getDataSource() {
// TODO: Fix this
SQLiteDataSource dataSource = new SQLiteDataSource();
dataSource.setUrl("jdbc:sqlite:" + dbName);
dataSource.setDatabaseName("meerkat"); //TODO: Make generic
return dataSource;
}
@ -98,10 +209,9 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi
+ " REFERENCES MsgTable(EntryNum), FOREIGN KEY (TagId) REFERENCES TagTable(TagId), UNIQUE (EntryNum, TagID))");
list.add("CREATE TABLE IF NOT EXISTS SignatureTable (EntryNum INTEGER, SignerId BLOB, Signature BLOB,"
+ " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))");
+ " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum), UNIQUE(SignerId, EntryNum))");
list.add("CREATE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId)");
list.add("CREATE UNIQUE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId, EntryNum)");
return list;
}

32
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataCallbackHandler.java Normal file
View File

@ -0,0 +1,32 @@
package meerkat.bulletinboard.sqlserver.mappers;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.comm.MessageOutputStream;
import meerkat.protobuf.BulletinBoardAPI.BatchData;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.RowMapper;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 19-Dec-15.
*/
public class BatchDataCallbackHandler implements RowCallbackHandler {
private final MessageOutputStream<BatchData> out;
public BatchDataCallbackHandler(MessageOutputStream<BatchData> out) {
this.out = out;
}
@Override
public void processRow(ResultSet rs) throws SQLException {
try {
out.writeMessage(BatchData.parseFrom(rs.getBytes(1)));
} catch (IOException e) {
//TODO: Log
}
}
}

26
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataMapper.java Normal file
View File

@ -0,0 +1,26 @@
package meerkat.bulletinboard.sqlserver.mappers;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.protobuf.BulletinBoardAPI.BatchData;
import org.springframework.jdbc.core.RowMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 19-Dec-15.
*/
public class BatchDataMapper implements RowMapper<BatchData> {
@Override
public BatchData mapRow(ResultSet rs, int rowNum) throws SQLException {
try {
return BatchData.parseFrom(rs.getBytes(1));
} catch (InvalidProtocolBufferException e) {
return BatchData.getDefaultInstance();
}
}
}

2
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/EntryNumMapper.java → bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/LongMapper.java
View File

@ -9,7 +9,7 @@ import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 11-Dec-15.
*/
public class EntryNumMapper implements RowMapper<Long> {
public class LongMapper implements RowMapper<Long> {
@Override
public Long mapRow(ResultSet rs, int rowNum) throws SQLException {

80
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageCallbackHandler.java Normal file
View File

@ -0,0 +1,80 @@
package meerkat.bulletinboard.sqlserver.mappers;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.*;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider.*;
import meerkat.comm.MessageOutputStream;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 21-Feb-16.
*/
public class MessageCallbackHandler implements RowCallbackHandler {
NamedParameterJdbcTemplate jdbcTemplate;
SQLQueryProvider sqlQueryProvider;
MessageOutputStream<BulletinBoardMessage> out;
public MessageCallbackHandler(NamedParameterJdbcTemplate jdbcTemplate, SQLQueryProvider sqlQueryProvider, MessageOutputStream<BulletinBoardMessage> out) {
this.jdbcTemplate = jdbcTemplate;
this.sqlQueryProvider = sqlQueryProvider;
this.out = out;
}
@Override
public void processRow(ResultSet rs) throws SQLException {
BulletinBoardMessage.Builder result;
try {
result = BulletinBoardMessage.newBuilder()
.setEntryNum(rs.getLong(1))
.setMsg(UnsignedBulletinBoardMessage.parseFrom(rs.getBytes(2)));
} catch (InvalidProtocolBufferException e) {
//TODO: log
return;
}
// Retrieve signatures
MapSqlParameterSource sqlParameterSource = new MapSqlParameterSource();
sqlParameterSource.addValue(QueryType.GET_SIGNATURES.getParamName(0), result.getEntryNum());
List<Crypto.Signature> signatures = jdbcTemplate.query(
sqlQueryProvider.getSQLString(QueryType.GET_SIGNATURES),
sqlParameterSource,
new SignatureMapper());
// Append signatures
result.addAllSig(signatures);
// Finalize message and add to message list.
try {
out.writeMessage(result.build());
} catch (IOException e) {
//TODO: log
e.printStackTrace();
}
}
}

31
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubMapper.java Normal file
View File

@ -0,0 +1,31 @@
package meerkat.bulletinboard.sqlserver.mappers;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.protobuf.BulletinBoardAPI.UnsignedBulletinBoardMessage;
import meerkat.util.BulletinBoardUtils;
import org.springframework.jdbc.core.RowMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 11-Dec-15.
*/
public class MessageStubMapper implements RowMapper<BulletinBoardMessage> {
@Override
public BulletinBoardMessage mapRow(ResultSet rs, int rowNum) throws SQLException {
return BulletinBoardMessage.newBuilder()
.setEntryNum(rs.getLong(1))
.setMsg(UnsignedBulletinBoardMessage.newBuilder()
.setData(ByteString.copyFrom(rs.getBytes(2)))
.setTimestamp(BulletinBoardUtils.toTimestampProto(rs.getTimestamp(3)))
.build())
.build();
}
}

18
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/StringMapper.java Normal file
View File

@ -0,0 +1,18 @@
package meerkat.bulletinboard.sqlserver.mappers;
import org.springframework.jdbc.core.RowMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 20-Dec-15.
*/
public class StringMapper implements RowMapper<String> {
@Override
public String mapRow(ResultSet rs, int rowNum) throws SQLException {
return rs.getString(1);
}
}

168
bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java
View File

@ -3,13 +3,10 @@ package meerkat.bulletinboard.webapp;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.StreamingOutput;
import meerkat.bulletinboard.BulletinBoardServer;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer;
@ -17,13 +14,20 @@ import meerkat.bulletinboard.sqlserver.H2QueryProvider;
import meerkat.bulletinboard.sqlserver.MySQLQueryProvider;
import meerkat.bulletinboard.sqlserver.SQLiteQueryProvider;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.BoolMsg;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessageList;
import meerkat.protobuf.BulletinBoardAPI.MessageFilterList;
import meerkat.rest.Constants;
import meerkat.comm.MessageOutputStream;
import meerkat.protobuf.BulletinBoardAPI;
import meerkat.protobuf.BulletinBoardAPI.*;
import static meerkat.bulletinboard.BulletinBoardConstants.*;
import static meerkat.rest.Constants.*;
@Path(Constants.BULLETIN_BOARD_SERVER_PATH)
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
/**
* An implementation of the BulletinBoardServer which functions as a WebApp
*/
@Path(BULLETIN_BOARD_SERVER_PATH)
public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextListener{
private static final String BULLETIN_BOARD_ATTRIBUTE_NAME = "bulletinBoard";
@ -79,24 +83,146 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL
}
}
@Path(Constants.POST_MESSAGE_PATH)
@Path(POST_MESSAGE_PATH)
@POST
@Consumes(Constants.MEDIATYPE_PROTOBUF)
@Produces(Constants.MEDIATYPE_PROTOBUF)
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException {
init();
return bulletinBoard.postMessage(msg);
}
@Path(Constants.READ_MESSAGES_PATH)
@POST
@Consumes(Constants.MEDIATYPE_PROTOBUF)
@Produces(Constants.MEDIATYPE_PROTOBUF)
@Override
public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException {
public void readMessages(MessageFilterList filterList, MessageOutputStream<BulletinBoardMessage> out) throws CommunicationException {
init();
return bulletinBoard.readMessages(filterList);
bulletinBoard.readMessages(filterList, out);
}
@Path(READ_MESSAGES_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
/**
* Wrapper for the readMessages method which streams the output into the response
*/
public StreamingOutput readMessages(final MessageFilterList filterList) {
return new StreamingOutput() {
@Override
public void write(OutputStream output) throws IOException, WebApplicationException {
MessageOutputStream<BulletinBoardMessage> out = new MessageOutputStream<>(output);
try {
init();
bulletinBoard.readMessages(filterList, out);
} catch (CommunicationException e) {
//TODO: Log
out.writeMessage(null);
}
}
};
}
@Path(BEGIN_BATCH_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public BoolMsg beginBatch(BeginBatchMessage message) {
try {
init();
return bulletinBoard.beginBatch(message);
} catch (CommunicationException e) {
System.err.println(e.getMessage());
return null;
}
}
@Path(POST_BATCH_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public BoolMsg postBatchMessage(BatchMessage batchMessage) {
try {
init();
return bulletinBoard.postBatchMessage(batchMessage);
} catch (CommunicationException e) {
System.err.println(e.getMessage());
return null;
}
}
@Path(CLOSE_BATCH_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public BoolMsg closeBatchMessage(CloseBatchMessage message) {
try {
init();
return bulletinBoard.closeBatchMessage(message);
} catch (CommunicationException e) {
System.err.println(e.getMessage());
return null;
}
}
@Override
public void readBatch(BatchSpecificationMessage message, MessageOutputStream<BatchData> out) {
try {
init();
bulletinBoard.readBatch(message, out);
} catch (CommunicationException | IllegalArgumentException e) {
System.err.println(e.getMessage());
}
}
@Path(READ_BATCH_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
/**
* Wrapper for the readBatch method which streams the output into the response
*/
public StreamingOutput readBatch(final BatchSpecificationMessage message) {
return new StreamingOutput() {
@Override
public void write(OutputStream output) throws IOException, WebApplicationException {
MessageOutputStream<BatchData> out = new MessageOutputStream<>(output);
try {
init();
bulletinBoard.readBatch(message, out);
} catch (CommunicationException e) {
//TODO: Log
out.writeMessage(null);
}
}
};
}
@Path(SYNC_QUERY_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException {
try{
init();
return bulletinBoard.querySync(syncQuery);
} catch (CommunicationException | IllegalArgumentException e) {
System.err.println(e.getMessage());
return null;
}
}
@Override

2
bulletin-board-server/src/main/webapp/WEB-INF/web.xml
View File

@ -31,7 +31,7 @@
<param-value>mypass</param-value></context-param>
<context-param>
<param-name>dbType</param-name>
<param-value>SQLite</param-value></context-param>
<param-value>H2</param-value></context-param>
<listener>
<listener-class>meerkat.bulletinboard.webapp.BulletinBoardWebApp</listener-class>
</listener>

47
bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java
View File

@ -4,8 +4,11 @@ package meerkat.bulletinboard;
import com.google.protobuf.ByteString;
import com.google.protobuf.TextFormat;
import com.google.protobuf.Timestamp;
import meerkat.comm.MessageInputStream;
import meerkat.protobuf.Crypto.*;
import meerkat.protobuf.BulletinBoardAPI.*;
import static meerkat.bulletinboard.BulletinBoardConstants.*;
import meerkat.rest.Constants;
import meerkat.rest.ProtobufMessageBodyReader;
import meerkat.rest.ProtobufMessageBodyWriter;
@ -17,7 +20,10 @@ import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.InputStream;
import java.util.List;
public class BulletinBoardSQLServerIntegrationTest {
@ -43,6 +49,16 @@ public class BulletinBoardSQLServerIntegrationTest {
byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24};
byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0};
Timestamp t1 = Timestamp.newBuilder()
.setSeconds(8276482)
.setNanos(4314)
.build();
Timestamp t2 = Timestamp.newBuilder()
.setSeconds(987591)
.setNanos(1513)
.build();
WebTarget webTarget;
Response response;
BoolMsg bool;
@ -50,12 +66,12 @@ public class BulletinBoardSQLServerIntegrationTest {
BulletinBoardMessage msg;
MessageFilterList filterList;
BulletinBoardMessageList msgList;
List<BulletinBoardMessage> msgList;
// Test writing mechanism
System.err.println("******** Testing: " + Constants.POST_MESSAGE_PATH);
webTarget = client.target(BASE_URL).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.POST_MESSAGE_PATH);
System.err.println("******** Testing: " + POST_MESSAGE_PATH);
webTarget = client.target(BASE_URL).path(BULLETIN_BOARD_SERVER_PATH).path(POST_MESSAGE_PATH);
System.err.println(webTarget.getUri());
msg = BulletinBoardMessage.newBuilder()
@ -63,6 +79,7 @@ public class BulletinBoardSQLServerIntegrationTest {
.addTag("Signature")
.addTag("Trustee")
.setData(ByteString.copyFrom(b1))
.setTimestamp(t1)
.build())
.addSig(Signature.newBuilder()
.setType(SignatureType.DSA)
@ -86,6 +103,7 @@ public class BulletinBoardSQLServerIntegrationTest {
.addTag("Vote")
.addTag("Trustee")
.setData(ByteString.copyFrom(b4))
.setTimestamp(t2)
.build())
.addSig(Signature.newBuilder()
.setType(SignatureType.ECDSA)
@ -101,8 +119,8 @@ public class BulletinBoardSQLServerIntegrationTest {
// Test reading mechanism
System.err.println("******** Testing: " + Constants.READ_MESSAGES_PATH);
webTarget = client.target(BASE_URL).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH);
System.err.println("******** Testing: " + READ_MESSAGES_PATH);
webTarget = client.target(BASE_URL).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH);
filterList = MessageFilterList.newBuilder()
.addFilter(
MessageFilter.newBuilder()
@ -112,13 +130,20 @@ public class BulletinBoardSQLServerIntegrationTest {
)
.build();
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF));
System.err.println(response);
msgList = response.readEntity(BulletinBoardMessageList.class);
System.err.println("List size: " + msgList.getMessageCount());
InputStream in = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF), InputStream.class);
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BulletinBoardMessage.class);
msgList = inputStream.asList();
System.err.println("List size: " + msgList.size());
System.err.println("This is the list:");
System.err.println(TextFormat.printToString(msgList));
assert msgList.getMessageCount() == 1;
for (BulletinBoardMessage message : msgList) {
System.err.println(TextFormat.printToString(message));
}
assert msgList.size() == 1;
}
}

382
bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java
View File

@ -1,9 +1,12 @@
package meerkat.bulletinboard;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.lang.reflect.InvocationTargetException;
import java.math.BigInteger;
import java.security.InvalidKeyException;
import java.security.KeyStore;
@ -12,26 +15,30 @@ import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.util.List;
import java.util.Random;
import java.util.*;
import com.google.protobuf.ByteString;
import com.google.protobuf.Timestamp;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageInputStream;
import meerkat.comm.MessageOutputStream;
import meerkat.comm.MessageInputStream.MessageInputStreamFactory;
import meerkat.crypto.Digest;
import meerkat.crypto.concrete.ECDSASignature;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.protobuf.BulletinBoardAPI.FilterType;
import meerkat.protobuf.BulletinBoardAPI.MessageFilter;
import meerkat.protobuf.BulletinBoardAPI.MessageFilterList;
import meerkat.protobuf.BulletinBoardAPI.UnsignedBulletinBoardMessage;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.util.BulletinBoardMessageGenerator;
import org.h2.util.DateTimeUtils;
import static org.junit.Assert.*;
import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.MatcherAssert.assertThat;
public class GenericBulletinBoardServerTest {
protected BulletinBoardServer bulletinBoardServer;
private ECDSASignature signers[];
private GenericBatchDigitalSignature[] signers;
private ByteString[] signerIDs;
private Random random;
@ -51,8 +58,14 @@ public class GenericBulletinBoardServerTest {
private String[] tags;
private byte[][] data;
private List<CompleteBatch> completeBatches;
private final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); // Used to time the tests
private BulletinBoardMessageGenerator bulletinBoardMessageGenerator;
private Digest digest;
/**
* @param bulletinBoardServer is an initialized server.
* @throws InstantiationException
@ -71,10 +84,10 @@ public class GenericBulletinBoardServerTest {
this.bulletinBoardServer = bulletinBoardServer;
signers = new ECDSASignature[2];
signers = new GenericBatchDigitalSignature[2];
signerIDs = new ByteString[signers.length];
signers[0] = new ECDSASignature();
signers[1] = new ECDSASignature();
signers[0] = new GenericBatchDigitalSignature(new ECDSASignature());
signers[1] = new GenericBatchDigitalSignature(new ECDSASignature());
InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE);
char[] password = KEYFILE_PASSWORD1.toCharArray();
@ -115,12 +128,21 @@ public class GenericBulletinBoardServerTest {
System.err.println("Couldn't find signing key " + e.getMessage());
fail("Couldn't find signing key " + e.getMessage());
}
random = new Random(0); // We use insecure randomness in tests for repeatability
// We use insecure randomness in tests for repeatability
random = new Random(0);
bulletinBoardMessageGenerator = new BulletinBoardMessageGenerator(random);
digest = new SHA256Digest();
long end = threadBean.getCurrentThreadCpuTime();
System.err.println("Finished initializing GenericBulletinBoardServerTest");
System.err.println("Time of operation: " + (end - start));
// Initialize Batch variables
completeBatches = new ArrayList<CompleteBatch>(10);
}
private byte randomByte(){
@ -170,7 +192,11 @@ public class GenericBulletinBoardServerTest {
for (i = 1; i <= MESSAGE_NUM; i++) {
unsignedMsgBuilder = UnsignedBulletinBoardMessage.newBuilder()
.setData(ByteString.copyFrom(data[i - 1]));
.setData(ByteString.copyFrom(data[i - 1]))
.setTimestamp(Timestamp.newBuilder()
.setSeconds(i)
.setNanos(i)
.build());
// Add tags based on bit-representation of message number.
@ -230,28 +256,39 @@ public class GenericBulletinBoardServerTest {
System.err.println("Starting to test tag and signature mechanism");
long start = threadBean.getCurrentThreadCpuTime();
List<BulletinBoardMessage> messages;
List<BulletinBoardMessage> messages = new LinkedList<>();
// Check tag mechanism
for (int i = 0 ; i < TAG_NUM ; i++){
// Retrieve messages having tag i
try {
messages = bulletinBoardServer.readMessages(
MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag(tags[i])
.build()
)
.build()
)
.getMessageList();
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag(tags[i])
.build()
)
.build();
} catch (CommunicationException e) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
bulletinBoardServer.readMessages(filterList, new MessageOutputStream<BulletinBoardMessage>(outputStream));
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BulletinBoardMessage.class);
messages = inputStream.asList();
} catch (CommunicationException | IOException e) {
fail(e.getMessage());
return;
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
fail(e.getMessage());
return;
}
@ -328,11 +365,26 @@ public class GenericBulletinBoardServerTest {
);
try {
messages = bulletinBoardServer.readMessages(filterListBuilder.build()).getMessageList();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
bulletinBoardServer.readMessages(filterListBuilder.build(), new MessageOutputStream<BulletinBoardMessage>(outputStream));
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BulletinBoardMessage.class);
messages = inputStream.asList();
} catch (CommunicationException e) {
System.err.println("Failed retrieving multi-tag messages from DB: " + e.getMessage());
fail("Failed retrieving multi-tag messages from DB: " + e.getMessage());
return;
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | IOException e) {
System.err.println("Falied to read from stream while retrieving multi-tag messages: " + e.getMessage());
fail("Falied to read from stream while retrieving multi-tag messages: " + e.getMessage());
return;
}
expectedMsgCount /= 2;
@ -359,11 +411,26 @@ public class GenericBulletinBoardServerTest {
.build());
try {
messages = bulletinBoardServer.readMessages(filterListBuilder.build()).getMessageList();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
bulletinBoardServer.readMessages(filterListBuilder.build(), new MessageOutputStream<BulletinBoardMessage>(outputStream));
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BulletinBoardMessage.class);
messages = inputStream.asList();
} catch (CommunicationException e) {
System.err.println("Failed retrieving multi-signature message from DB: " + e.getMessage());
fail("Failed retrieving multi-signature message from DB: " + e.getMessage());
return;
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | IOException e) {
System.err.println("Falied to read from stream while retrieving multi-signature message: " + e.getMessage());
fail("Falied to read from stream while retrieving multi-signature message: " + e.getMessage());
return;
}
assertThat(messages.size(), is(MESSAGE_NUM / 4));
@ -377,6 +444,263 @@ public class GenericBulletinBoardServerTest {
System.err.println("Time of operation: " + (end - start));
}
/**
* Tests that posting a message before opening a batch does not work
* @throws CommunicationException
*/
public void testBatchPostAfterClose() throws CommunicationException, SignatureException {
final int BATCH_ID = 100;
CompleteBatch completeBatch = new CompleteBatch(Timestamp.newBuilder()
.setSeconds(978325)
.setNanos(8097234)
.build());
BoolMsg result;
// Create data
completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder()
.setSignerId(signerIDs[1])
.setBatchId(BATCH_ID)
.addTag("Test")
.build());
BatchData batchData = BatchData.newBuilder()
.setData(ByteString.copyFrom((new byte[] {1,2,3,4})))
.build();
completeBatch.appendBatchData(batchData);
signers[1].updateContent(completeBatch);
completeBatch.setSignature(signers[1].sign());
// Begin batch
result = bulletinBoardServer.beginBatch(completeBatch.getBeginBatchMessage());
assertThat("Was not able to open batch", result.getValue(), is(true));
// Post data
BatchMessage batchMessage = BatchMessage.newBuilder()
.setSignerId(signerIDs[1])
.setBatchId(BATCH_ID)
.setData(batchData)
.build();
result = bulletinBoardServer.postBatchMessage(batchMessage);
assertThat("Was not able to post batch message", result.getValue(), is(true));
// Close batch
result = bulletinBoardServer.closeBatchMessage(completeBatch.getCloseBatchMessage());
assertThat("Was not able to close batch", result.getValue(), is(true));
// Attempt to open batch again
result = bulletinBoardServer.beginBatch(completeBatch.getBeginBatchMessage());
assertThat("Was able to open a closed batch", result.getValue(), is(false));
// Attempt to add batch data
result = bulletinBoardServer.postBatchMessage(batchMessage);
assertThat("Was able to change a closed batch", result.getValue(), is(false));
}
/**
* Posts a complete batch message
* @throws CommunicationException
*/
public void testPostBatch() throws CommunicationException, SignatureException {
CompleteBatch completeBatch = new CompleteBatch(Timestamp.newBuilder()
.setSeconds(12345)
.setNanos(1111)
.build());
int currentBatch = completeBatches.size();
BoolMsg result;
// Define batch data
String[] tempBatchTags = new String[]{randomString(),randomString(),randomString()};
byte[][] tempBatchData = new byte[Math.abs(randomByte())][];
for (int i = 0 ; i < tempBatchData.length ; i++) {
tempBatchData[i] = new byte[Math.abs(randomByte())];
for (int j = 0; j < tempBatchData[i].length; j++) {
tempBatchData[i][j] = randomByte();
}
}
// Begin batch
completeBatch.setBeginBatchMessage(BeginBatchMessage.newBuilder()
.setSignerId(signerIDs[0])
.setBatchId(currentBatch)
.addAllTag(Arrays.asList(tempBatchTags))
.build());
result = bulletinBoardServer.beginBatch(completeBatch.getBeginBatchMessage());
assertThat("Could not begin batch " + currentBatch, result.getValue(), is(true));
// Add batch data and randomize data posting order
List<Integer> dataOrder = new ArrayList<Integer>(tempBatchData.length);
for (int i = 0 ; i < tempBatchData.length ; i++) {
dataOrder.add(i);
completeBatch.appendBatchData(BatchData.newBuilder()
.setData(ByteString.copyFrom(tempBatchData[i]))
.build());
}
Collections.shuffle(dataOrder);
// Post data
for (int i = 0 ; i < tempBatchData.length ; i++) {
int dataIndex = dataOrder.get(i);
result = bulletinBoardServer.postBatchMessage(BatchMessage.newBuilder()
.setSignerId(signerIDs[0])
.setBatchId(currentBatch)
.setSerialNum(dataIndex)
.setData(completeBatch.getBatchDataList().get(dataIndex))
.build());
assertThat("Could not post batch data for batch ID " + currentBatch + " serial number " + dataIndex,
result.getValue(), is(true));
}
// Close batch
signers[0].updateContent(completeBatch);
completeBatch.setSignature(signers[0].sign());
result = bulletinBoardServer.closeBatchMessage(completeBatch.getCloseBatchMessage());
assertThat("Could not close batch " + currentBatch, result.getValue(), is(true));
// Update locally stored batches
completeBatches.add(completeBatch);
}
public void testReadBatch() throws CommunicationException {
for (CompleteBatch completeBatch : completeBatches) {
try {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
BatchSpecificationMessage batchSpecificationMessage =
BatchSpecificationMessage.newBuilder()
.setSignerId(completeBatch.getBeginBatchMessage().getSignerId())
.setBatchId(completeBatch.getBeginBatchMessage().getBatchId())
.setStartPosition(0)
.build();
bulletinBoardServer.readBatch(batchSpecificationMessage, new MessageOutputStream<BatchData>(outputStream));
MessageInputStream<BatchData> inputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BatchData.class);
List<BatchData> batchDataList = inputStream.asList();
assertThat("Non-matching batch data for batch " + completeBatch.getBeginBatchMessage().getBatchId(),
completeBatch.getBatchDataList().equals(batchDataList), is(true));
} catch (IOException | NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
assertThat("Error reading batch data list from input stream", false);
}
}
}
public void testSyncQuery()
throws SignatureException, CommunicationException, IOException,NoSuchMethodException, IllegalAccessException, InvocationTargetException {
Checksum checksum = new SimpleChecksum();
Timestamp timestamp = Timestamp.newBuilder()
.setSeconds(1)
.setNanos(0)
.build();
BulletinBoardMessage newMessage = bulletinBoardMessageGenerator.generateRandomMessage(signers, timestamp, 10, 10);
BoolMsg result = bulletinBoardServer.postMessage(newMessage);
assertThat("Failed to post message to BB Server", result.getValue(), is(true));
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
digest.update(newMessage.getMsg());
ByteString messageID = ByteString.copyFrom(digest.digest());
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(messageID)
.build())
.build();
bulletinBoardServer.readMessages(filterList, new MessageOutputStream<BulletinBoardMessage>(outputStream));
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BulletinBoardMessage.class);
long lastEntry = inputStream.asList().get(0).getEntryNum();
SyncQuery syncQuery = SyncQuery.newBuilder()
.setFilterList(MessageFilterList.getDefaultInstance())
.addQuery(SingleSyncQuery.newBuilder()
.setChecksum(2)
.setTimeOfSync(Timestamp.newBuilder()
.setSeconds(2)
.setNanos(0)
.build())
.build())
.build();
SyncQueryResponse queryResponse = bulletinBoardServer.querySync(syncQuery);
assertThat("Sync query replies with positive sync when no sync was expected", queryResponse.getLastEntryNum(), is(equalTo(-1l)));
syncQuery = SyncQuery.newBuilder()
.setFilterList(MessageFilterList.getDefaultInstance())
.addQuery(SingleSyncQuery.newBuilder()
.setChecksum(checksum.getChecksum(messageID))
.setTimeOfSync(timestamp)
.build())
.build();
queryResponse = bulletinBoardServer.querySync(syncQuery);
assertThat("Sync query reply contained wrong last entry number", lastEntry, is(equalTo(queryResponse.getLastEntryNum())));
assertThat("Sync query reply contained wrong timestamp", timestamp, is(equalTo(queryResponse.getLastTimeOfSync())));
}
public void close(){
signers[0].clearSigningKey();

33
bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java
View File

@ -107,6 +107,39 @@ public class H2BulletinBoardServerTest {
System.err.println("Time of operation: " + (end - start));
}
@Test
public void testBatchPostAfterClose() {
try{
serverTest.testBatchPostAfterClose();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@Test
public void testBatch() {
final int BATCH_NUM = 20;
try{
for (int i = 0 ; i < BATCH_NUM ; i++) {
serverTest.testPostBatch();
}
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
try{
serverTest.testReadBatch();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@After
public void close() {
System.err.println("Starting to close H2BulletinBoardServerTest");

46
bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java
View File

@ -8,8 +8,11 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.lang.reflect.InvocationTargetException;
import java.security.SignatureException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@ -111,6 +114,49 @@ public class MySQLBulletinBoardServerTest {
System.err.println("Time of operation: " + (end - start));
}
@Test
public void testBatchPostAfterClose() {
try{
serverTest.testBatchPostAfterClose();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@Test
public void testBatch() {
final int BATCH_NUM = 20;
try{
for (int i = 0 ; i < BATCH_NUM ; i++) {
serverTest.testPostBatch();
}
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
try{
serverTest.testReadBatch();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@Test
public void testSyncQuery() {
try {
serverTest.testSyncQuery();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@After
public void close() {
System.err.println("Starting to close MySQLBulletinBoardServerTest");

3
meerkat-common/build.gradle
View File

@ -39,6 +39,7 @@ version += "${isSnapshot ? '-SNAPSHOT' : ''}"
dependencies {
// Logging
compile 'org.slf4j:slf4j-api:1.7.7'
compile 'javax.ws.rs:javax.ws.rs-api:2.0.+'
runtime 'ch.qos.logback:logback-classic:1.1.2'
runtime 'ch.qos.logback:logback-core:1.1.2'
@ -46,7 +47,7 @@ dependencies {
compile 'com.google.protobuf:protobuf-java:3.+'
// ListeningExecutor
compile 'com.google.guava:guava:11.0.+'
compile 'com.google.guava:guava:15.0'
// Crypto
compile 'org.factcenter.qilin:qilin:1.2+'

110
meerkat-common/src/main/java/meerkat/bulletinboard/AsyncBulletinBoardClient.java Normal file
View File

@ -0,0 +1,110 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.ByteString;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 14-Dec-15.
*/
public interface AsyncBulletinBoardClient extends BulletinBoardClient {
public interface MessageHandler {
void handleNewMessages(List<BulletinBoardMessage> messageList);
}
/**
* Post a message to the bulletin board in an asynchronous manner
* @param msg is the message to be posted
* @param callback is a class containing methods to handle the result of the operation
* @return a unique message ID for the message, that can be later used to retrieve the batch
*/
public MessageID postMessage(BulletinBoardMessage msg, FutureCallback<Boolean> callback);
/**
* Perform an end-to-end post of a signed batch message
* @param completeBatch contains all the data of the batch including the meta-data and the signature
* @param callback is a class containing methods to handle the result of the operation
* @return a unique identifier for the batch message
*/
public MessageID postBatch(CompleteBatch completeBatch, FutureCallback<Boolean> callback);
/**
* This message informs the server about the existence of a new batch message and supplies it with the tags associated with it
* @param beginBatchMessage contains the data required to begin the batch
* @param callback is a callback function class for handling results of the operation
*/
public void beginBatch(BeginBatchMessage beginBatchMessage, FutureCallback<Boolean> callback);
/**
* This method posts batch data into an (assumed to be open) batch
* It does not close the batch
* @param signerId is the canonical form for the ID of the sender of this batch
* @param batchId is a unique (per signer) ID for this batch
* @param batchDataList is the (canonically ordered) list of data comprising the portion of the batch to be posted
* @param startPosition is the location (in the batch) of the first entry in batchDataList
* (optionally used to continue interrupted post operations)
* The first position in the batch is position 0
* @param callback is a callback function class for handling results of the operation
*/
public void postBatchData(byte[] signerId, int batchId, List<BatchData> batchDataList,
int startPosition, FutureCallback<Boolean> callback);
/**
* Overloading of the postBatchData method which starts at the first position in the batch
*/
public void postBatchData(byte[] signerId, int batchId, List<BatchData> batchDataList, FutureCallback<Boolean> callback);
/**
* Overloading of the postBatchData method which uses ByteString
*/
public void postBatchData(ByteString signerId, int batchId, List<BatchData> batchDataList,
int startPosition, FutureCallback<Boolean> callback);
/**
* Overloading of the postBatchData method which uses ByteString and starts at the first position in the batch
*/
public void postBatchData(ByteString signerId, int batchId, List<BatchData> batchDataList, FutureCallback<Boolean> callback);
/**
* Attempts to close a batch message
* @param closeBatchMessage contains the data required to close the batch
* @param callback is a callback function class for handling results of the operation
*/
public void closeBatch(CloseBatchMessage closeBatchMessage, FutureCallback<Boolean> callback);
/**
* Check how "safe" a given message is in an asynchronous manner
* The result of the computation is a rank between 0.0 and 1.0 indicating the fraction of servers containing the message
* @param id is the unique message identifier for retrieval
* @param callback is a callback function class for handling results of the operation
*/
public void getRedundancy(MessageID id, FutureCallback<Float> callback);
/**
* Read all messages posted matching the given filter in an asynchronous manner
* Note that if messages haven't been "fully posted", this might return a different
* set of messages in different calls. However, messages that are fully posted
* are guaranteed to be included.
* @param filterList return only messages that match the filters (null means no filtering).
* @param callback is a callback function class for handling results of the operation
*/
public void readMessages(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback);
/**
* Read a given batch message from the bulletin board
* @param batchSpecificationMessage contains the data required to specify a single batch instance
* @param callback is a callback class for handling the result of the operation
*/
public void readBatch(BatchSpecificationMessage batchSpecificationMessage, FutureCallback<CompleteBatch> callback);
/**
* Subscribes to a notifier that will return any new messages on the server that match the given filters
* @param filterList defines the set of filters for message retrieval
* @param messageHandler defines the handler for new messages received
*/
public void subscribe(MessageFilterList filterList, MessageHandler messageHandler);
}

20
meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigest.java Normal file
View File

@ -0,0 +1,20 @@
package meerkat.bulletinboard;
import meerkat.crypto.Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 18-Dec-15.
* Extends the Digest interface with a method for digesting Batch messages
*/
public interface BatchDigest extends Digest {
/**
* Update the digest with the batch message data (ignore the signature)
* @param completeBatch is the batch message that needs to be digested
*/
public void update(CompleteBatch completeBatch);
}

34
meerkat-common/src/main/java/meerkat/bulletinboard/BatchDigitalSignature.java Normal file
View File

@ -0,0 +1,34 @@
package meerkat.bulletinboard;
import meerkat.crypto.DigitalSignature;
import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage;
import meerkat.protobuf.BulletinBoardAPI.BatchData;
import meerkat.protobuf.Crypto.Signature;
import java.security.InvalidKeyException;
import java.security.SignatureException;
import java.security.cert.CertificateException;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 20-Dec-15.
* Extends the DigitalSignature interface with methods for signing and authenticating Batch messages
*/
public interface BatchDigitalSignature extends DigitalSignature {
/**
* Appends the batch data to the signed content (ignoring the signature)
* @param completeBatch contains all the data about the batch
* @throws SignatureException
*/
public void updateContent(CompleteBatch completeBatch) throws SignatureException;
/**
* Performs a complete verification process on the given batch message
* @param completeBatch contains the batch data as well as the signature
* @return TRUE if the batch is verified and FALSE otherwise
* @throws SignatureException | SignatureException | InvalidKeyException when underlying methods do so
*/
public boolean verify(CompleteBatch completeBatch) throws SignatureException, CertificateException, InvalidKeyException;
}

28
meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardClient.java
View File

@ -1,6 +1,6 @@
package meerkat.bulletinboard;
import meerkat.comm.*;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.Voting.*;
import static meerkat.protobuf.BulletinBoardAPI.*;
@ -12,11 +12,6 @@ import java.util.List;
*/
public interface BulletinBoardClient {
interface ClientCallback<T> {
void handleCallback(T msg);
void handleFailure(Throwable t);
}
/**
* Initialize the client to use some specified servers
* @param clientParams contains the parameters required for the client setup
@ -24,26 +19,31 @@ public interface BulletinBoardClient {
void init(BulletinBoardClientParams clientParams);
/**
* Post a message to the bulletin board
* @param msg
* Post a message to the bulletin board in a synchronous manner
* @param msg is the message to be posted
* @return a unique message ID for the message, that can be later used to retrieve the batch
* @throws CommunicationException
*/
MessageID postMessage(BulletinBoardMessage msg, ClientCallback<?> callback);
MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException;
/**
* Check how "safe" a given message is
* @param id
* Check how "safe" a given message is in a synchronous manner
* @param id is the unique message identifier for retrieval
* @return a normalized "redundancy score" from 0 (local only) to 1 (fully published)
*/
void getRedundancy(MessageID id, ClientCallback<Float> callback);
float getRedundancy(MessageID id);
/**
* Read all messages posted matching the given filter
* Read all messages posted matching the given filter in a synchronous manner
* Note that if messages haven't been "fully posted", this might return a different
* set of messages in different calls. However, messages that are fully posted
* are guaranteed to be included.
* @param filterList return only messages that match the filters (null means no filtering).
* @return the list of messages
*/
void readMessages(MessageFilterList filterList, ClientCallback<List<BulletinBoardMessage>> callback);
List<BulletinBoardMessage> readMessages(MessageFilterList filterList);
/**
* Closes all connections, if any.

24
meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardConstants.java Normal file
View File

@ -0,0 +1,24 @@
package meerkat.bulletinboard;
/**
* Created by Arbel Deutsch Peled on 21-Dec-15.
*/
public interface BulletinBoardConstants {
// Relative addresses for Bulletin Board operations
public static final String BULLETIN_BOARD_SERVER_PATH = "/bbserver";
public static final String READ_MESSAGES_PATH = "/readmessages";
public static final String READ_BATCH_PATH = "/readbatch";
public static final String POST_MESSAGE_PATH = "/postmessage";
public static final String BEGIN_BATCH_PATH = "/beginbatch";
public static final String POST_BATCH_PATH = "/postbatch";
public static final String CLOSE_BATCH_PATH = "/closebatch";
public static final String SYNC_QUERY_PATH = "/syncquery";
// Other Constants
public static final String BATCH_TAG = "@BATCH";
public static final String BATCH_ID_TAG_PREFIX = "BATCHID#";
}

84
meerkat-common/src/main/java/meerkat/bulletinboard/BulletinBoardServer.java
View File

@ -1,42 +1,94 @@
package meerkat.bulletinboard;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageOutputStream;
import meerkat.protobuf.BulletinBoardAPI.*;
/**
* Created by Arbel on 07/11/15.
*
* This interface refers to a single instance of a Bulletin Board.
* An implementation of this interface may use any DB and be hosted on any machine.
* This interface refers to a single instance of a Bulletin Board
* An implementation of this interface may use any DB and be hosted on any machine.
*/
public interface BulletinBoardServer{
/**
* This method initializes the server by reading the signature data and storing it.
* It also establishes the connection to the DB.
* @throws CommunicationException on DB connection error.
* This method initializes the server by reading the signature data and storing it
* It also establishes the connection to the DB
* @throws CommunicationException on DB connection error
*/
public void init(String meerkatDB) throws CommunicationException;
/**
* Post a message to bulletin board.
* @param msg is the actual (signed) message
* @return TRUE if the message has been authenticated and FALSE otherwise (in ProtoBuf form).
* @throws CommunicationException on DB connection error.
* @return TRUE if the message has been authenticated and FALSE otherwise (in ProtoBuf form)
* @throws CommunicationException on DB connection error
*/
public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException;
public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException;
/**
* Read all messages posted matching the given filter.
* @param filter return only messages that match the filter (empty list means no filtering).
* @return
* Read all messages posted matching the given filter
* @param filterList return only messages that match the filters (empty list or null means no filtering)
* @param out is an output stream into which the matching messages are written
* @throws CommunicationException on DB connection error
*/
BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException;
public void readMessages(MessageFilterList filterList, MessageOutputStream<BulletinBoardMessage> out) throws CommunicationException;
/**
* Informs server about a new batch message
* @param message contains the required data about the new batch
* @return TRUE if the batch request is accepted amd FALSE otherwise
* Specifically, if such a batch already exists and is not yet closed: the value returned will be TRUE
* However, if such a batch exists and is already closed: the value returned will be FALSE
* @throws CommunicationException on DB connection error
*/
public BoolMsg beginBatch(BeginBatchMessage message) throws CommunicationException;
/**
* Posts a (part of a) batch message to the bulletin board
* Note that the existence and contents of a batch message are not available for reading before the batch is finalized
* @param batchMessage contains the (partial) data this message carries as well as meta-data required in order to place the data
* in the correct position inside the correct batch
* @return TRUE if the message is accepted and successfully saved and FALSE otherwise
* Specifically, if the batch is already closed: the value returned will be FALSE
* However, requiring to open a batch before insertion of messages is implementation-dependent
* @throws CommunicationException on DB connection error
*/
public BoolMsg postBatchMessage(BatchMessage batchMessage) throws CommunicationException;
/**
* Attempts to close and finalize a batch message
* @param message contains the data necessary to close the batch; in particular: the signature for the batch
* @return TRUE if the batch was successfully closed, FALSE otherwise
* Specifically, if the signature is invalid or if some of the batch parts have not yet been submitted: the value returned will be FALSE
* @throws CommunicationException on DB connection error
*/
public BoolMsg closeBatchMessage(CloseBatchMessage message) throws CommunicationException;
/**
* Reads a batch message from the server (starting with the supplied position)
* @param message specifies the signer ID and the batch ID to read as well as an (optional) start position
* @param out is a stream of the ordered batch messages starting from the specified start position (if given) or from the beginning (if omitted)
* @throws CommunicationException on DB connection error
* @throws IllegalArgumentException if message does not specify a batch
*/
public void readBatch(BatchSpecificationMessage message, MessageOutputStream<BatchData> out) throws CommunicationException, IllegalArgumentException;
/**
* Queries the database for sync status with respect to a given sync query
* @param syncQuery contains a succinct representation of states to compare to
* @return a SyncQueryResponse object containing the representation of the most recent state the database matches
* @throws CommunicationException
*/
public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException;
/**
* This method closes the connection to the DB.
* @throws CommunicationException on DB connection error.
* This method closes the connection to the DB
* @throws CommunicationException on DB connection error
*/
public void close() throws CommunicationException;
public void close() throws CommunicationException;
}

122
meerkat-common/src/main/java/meerkat/bulletinboard/Checksum.java Normal file
View File

@ -0,0 +1,122 @@
package meerkat.bulletinboard;
import com.google.protobuf.ByteString;
import meerkat.crypto.Digest;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.protobuf.BulletinBoardAPI.MessageID;
import java.util.Collection;
/**
* Created by Arbel Deutsch Peled on 01-Mar-16.
* This interface is used to create checksums of Bulletin Board messages IDs
* This is useful in comparing database states
*/
public interface Checksum {
/**
* Sets the Digest method which is used in creating message IDs from the messages themselves
* This method must be called with an initialized Digest before calling any methods that receive a parameter of type BulletinBoardMessage
* @param digest is the Digest that will be used to create message IDs from Bulletin Board Messages
*/
public void setDigest(Digest digest);
/**
* Used to reset the current checksum state
*/
public void reset();
/**
* Update the current checksum with the given ID
* @param messageID is the message ID to be added
*/
public void update(MessageID messageID);
/**
* Update the current checksum with the given collection of IDs
* @param messageIDs contains the message IDs
*/
public void update(Collection<MessageID> messageIDs);
/**
* Update the current checksum with the given ID
* @param messageID is the message ID to be added
*/
public void update(ByteString messageID);
/**
* Update the current checksum with the given ID
* @param messageID is the message ID to be added
*/
public void update(byte[] messageID);
/**
* Update the current checksum with the message ID of the given message
* @param bulletinBoardMessage is the message whose ID should be added to the checksum
* @throws IllegalStateException if a Digest has not been set before calling this method
*/
public void digestAndUpdate(BulletinBoardMessage bulletinBoardMessage) throws IllegalStateException;
/**
* Update the current checksum with the message IDs of the given messages
* @param bulletinBoardMessages contains the messages whose IDs should be added to the checksum
* @throws IllegalStateException if a Digest has not been set before calling this method
*/
public void digestAndUpdate(Collection<BulletinBoardMessage> bulletinBoardMessages) throws IllegalStateException;
/**
* Returns the current checksum without changing the checksum state
* @return the current checksum
*/
public long getChecksum();
/**
* Updates the current checksum with the given ID and returns the resulting checksum
* The checksum is not reset afterwards
* @param messageID is the message ID to be added
* @return the updated checksum
*/
public long getChecksum(MessageID messageID);
/**
* Updates the current checksum with the given ID and returns the resulting checksum
* The checksum is not reset afterwards
* @param messageID is the message ID to be added
* @return the updated checksum
*/
public long getChecksum(ByteString messageID);
/**
* Updates the current checksum with the given ID and returns the resulting checksum
* The checksum is not reset afterwards
* @param messageID is the message ID to be added
* @return the updated checksum
*/
public long getChecksum(byte[] messageID);
/**
* Updates the current checksum with the given IDs and returns the resulting checksum
* The checksum is not reset afterwards
* @param messageIDs contains the message IDs to be added
* @return the updated checksum
*/
public long getChecksum(Collection<MessageID> messageIDs);
/**
* Updates the current checksum with the message ID of the given message
* The checksum is not reset afterwards
* @param bulletinBoardMessage is the message whose ID should be added to the checksum
* @return the updated checksum
*/
public long digestAndGetChecksum(BulletinBoardMessage bulletinBoardMessage) throws IllegalStateException;
/**
* Updates the current checksum with the message IDs of the given messages
* The checksum is not reset afterwards
* @param bulletinBoardMessages contains the messages whose IDs should be added to the checksum
* @return the updated checksum
*/
public long digestAndGetChecksum(Collection<BulletinBoardMessage> bulletinBoardMessages) throws IllegalStateException;
}

145
meerkat-common/src/main/java/meerkat/bulletinboard/CompleteBatch.java Normal file
View File

@ -0,0 +1,145 @@
package meerkat.bulletinboard;
import com.google.protobuf.Timestamp;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto.*;
import meerkat.util.BulletinBoardMessageComparator;
import java.util.LinkedList;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 14-Dec-15.
*
* A data structure for holding a complete batch message along with its signature
*/
public class CompleteBatch {
private BeginBatchMessage beginBatchMessage;
private List<BatchData> batchDataList;
private Signature signature;
private Timestamp timestamp;
public CompleteBatch() {
batchDataList = new LinkedList<BatchData>();
}
public CompleteBatch(BeginBatchMessage newBeginBatchMessage) {
this();
beginBatchMessage = newBeginBatchMessage;
}
public CompleteBatch(BeginBatchMessage newBeginBatchMessage, List<BatchData> newDataList) {
this(newBeginBatchMessage);
appendBatchData(newDataList);
}
public CompleteBatch(BeginBatchMessage newBeginBatchMessage, List<BatchData> newDataList, Signature newSignature) {
this(newBeginBatchMessage, newDataList);
signature = newSignature;
}
public CompleteBatch(BeginBatchMessage newBeginBatchMessage, List<BatchData> newDataList, Signature newSignature, Timestamp timestamp) {
this(newBeginBatchMessage, newDataList, newSignature);
this.timestamp = timestamp;
}
public CompleteBatch(Timestamp timestamp) {
this();
this.timestamp = timestamp;
}
public BeginBatchMessage getBeginBatchMessage() {
return beginBatchMessage;
}
public List<BatchData> getBatchDataList() {
return batchDataList;
}
public Signature getSignature() {
return signature;
}
public Timestamp getTimestamp() {
return timestamp;
}
public CloseBatchMessage getCloseBatchMessage() {
return CloseBatchMessage.newBuilder()
.setBatchId(getBeginBatchMessage().getBatchId())
.setBatchLength(getBatchDataList().size())
.setSig(getSignature())
.setTimestamp(getTimestamp())
.build();
}
public void setBeginBatchMessage(BeginBatchMessage beginBatchMessage) {
this.beginBatchMessage = beginBatchMessage;
}
public void appendBatchData(BatchData newBatchData) {
batchDataList.add(newBatchData);
}
public void appendBatchData(List<BatchData> newBatchDataList) {
batchDataList.addAll(newBatchDataList);
}
public void setSignature(Signature newSignature) {
signature = newSignature;
}
public void setTimestamp(Timestamp timestamp) {
this.timestamp = timestamp;
}
@Override
public boolean equals(Object other) {
if (!(other instanceof CompleteBatch)) {
return false;
}
CompleteBatch otherBatch = (CompleteBatch) other;
boolean result = true;
if (beginBatchMessage == null) {
if (otherBatch.getBeginBatchMessage() != null)
return false;
} else {
result = result && beginBatchMessage.equals(otherBatch.getBeginBatchMessage());
}
if (batchDataList == null) {
if (otherBatch.getBatchDataList() != null)
return false;
} else {
result = result && batchDataList.equals(otherBatch.getBatchDataList());
}
if (signature == null) {
if (otherBatch.getSignature() != null)
return false;
} else {
result = result && signature.equals(otherBatch.getSignature());
}
if (timestamp == null) {
if (otherBatch.getTimestamp() != null)
return false;
} else {
result = result && timestamp.equals(otherBatch.getTimestamp());
}
return result;
}
@Override
public String toString() {
return "Batch " + beginBatchMessage.getSignerId().toString() + ":" + beginBatchMessage.getBatchId();
}
}

61
meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigest.java Normal file
View File

@ -0,0 +1,61 @@
package meerkat.bulletinboard;
import com.google.protobuf.Message;
import meerkat.crypto.Digest;
import meerkat.protobuf.BulletinBoardAPI.MessageID;
import meerkat.protobuf.BulletinBoardAPI.BatchData;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 19-Dec-15.
* Wrapper class for digesting Batches in a standardized way
*/
public class GenericBatchDigest implements BatchDigest{
private Digest digest;
public GenericBatchDigest(Digest digest) {
this.digest = digest;
}
@Override
public void update(CompleteBatch completeBatch) {
update(completeBatch.getBeginBatchMessage());
for (BatchData batchData : completeBatch.getBatchDataList()) {
update(batchData);
}
update(completeBatch.getTimestamp());
}
@Override
public byte[] digest() {
return digest.digest();
}
@Override
public MessageID digestAsMessageID() {
return digest.digestAsMessageID();
}
@Override
public void update(Message msg) {
digest.update(msg);
}
@Override
public void reset() {
digest.reset();
}
@Override
public GenericBatchDigest clone() throws CloneNotSupportedException{
return new GenericBatchDigest(digest.clone());
}
}

104
meerkat-common/src/main/java/meerkat/bulletinboard/GenericBatchDigitalSignature.java Normal file
View File

@ -0,0 +1,104 @@
package meerkat.bulletinboard;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
import meerkat.crypto.DigitalSignature;
import meerkat.protobuf.BulletinBoardAPI.BatchData;
import meerkat.protobuf.Crypto;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.SignatureException;
import java.security.NoSuchAlgorithmException;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
/**
* Created by Arbel Deutsch Peled on 20-Dec-15.
* Wrapper class for signing and verifying Batch signatures in a standardized way
*/
public class GenericBatchDigitalSignature implements BatchDigitalSignature{
private DigitalSignature digitalSignature;
public GenericBatchDigitalSignature(DigitalSignature digitalSignature) {
this.digitalSignature = digitalSignature;
}
@Override
public void updateContent(CompleteBatch completeBatch) throws SignatureException {
digitalSignature.updateContent(completeBatch.getBeginBatchMessage());
for (BatchData batchData : completeBatch.getBatchDataList()) {
digitalSignature.updateContent(batchData);
}
digitalSignature.updateContent(completeBatch.getTimestamp());
}
@Override
public boolean verify(CompleteBatch completeBatch) throws SignatureException, CertificateException, InvalidKeyException {
digitalSignature.initVerify(completeBatch.getSignature());
updateContent(completeBatch);
return digitalSignature.verify();
}
@Override
public void loadVerificationCertificates(InputStream certStream) throws CertificateException {
digitalSignature.loadVerificationCertificates(certStream);
}
@Override
public void clearVerificationCertificates() {
digitalSignature.clearVerificationCertificates();
}
@Override
public void updateContent(Message msg) throws SignatureException {
digitalSignature.updateContent(msg);
}
@Override
public Crypto.Signature sign() throws SignatureException {
return digitalSignature.sign();
}
@Override
public void initVerify(Crypto.Signature sig) throws CertificateException, InvalidKeyException {
digitalSignature.initVerify(sig);
}
@Override
public boolean verify() {
return digitalSignature.verify();
}
@Override
public KeyStore.Builder getPKCS12KeyStoreBuilder(InputStream keyStream, char[] password)
throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException {
return digitalSignature.getPKCS12KeyStoreBuilder(keyStream, password);
}
@Override
public void loadSigningCertificate(KeyStore.Builder keyStoreBuilder) throws IOException, CertificateException, UnrecoverableKeyException {
digitalSignature.loadSigningCertificate(keyStoreBuilder);
}
@Override
public ByteString getSignerID() {
return digitalSignature.getSignerID();
}
@Override
public void clearSigningKey() {
digitalSignature.clearSigningKey();
}
}

131
meerkat-common/src/main/java/meerkat/bulletinboard/SimpleChecksum.java Normal file
View File

@ -0,0 +1,131 @@
package meerkat.bulletinboard;
import com.google.protobuf.ByteString;
import meerkat.crypto.Digest;
import meerkat.protobuf.BulletinBoardAPI.MessageID;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import java.util.Collection;
/**
* Created by Arbel Deutsch Peled on 01-Mar-16.
* Implementation of Checksum via bitwise XOR of the bytes of message IDs
*/
public class SimpleChecksum implements Checksum{
private Digest digest;
private long checksum;
public SimpleChecksum() {
digest = null;
reset();
}
@Override
public void setDigest(Digest digest) {
this.digest = digest;
}
@Override
public void reset() {
checksum = 0;
}
@Override
public void update(MessageID messageID) {
ByteString messageIDByteString = messageID.getID();
update(messageIDByteString);
}
@Override
public void update(Collection<MessageID> messageIDs) {
for (MessageID messageID : messageIDs){
update(messageID);
}
}
@Override
public void update(ByteString messageID) {
for (int i = 0 ; i < messageID.size() ; i++){
checksum &= messageID.byteAt(i);
}
}
@Override
public void update(byte[] messageID) {
for (int i = 0 ; i < messageID.length ; i++){
checksum &= messageID[i];
}
}
private void checkDigest() throws IllegalStateException {
if (digest == null){
throw new IllegalStateException("Digest method not set. Use setDigest method before calling digestAndUpdate.");
}
}
@Override
public void digestAndUpdate(BulletinBoardMessage bulletinBoardMessage) throws IllegalStateException {
checkDigest();
digest.reset();
digest.update(bulletinBoardMessage);
update(digest.digest());
}
@Override
public void digestAndUpdate(Collection<BulletinBoardMessage> bulletinBoardMessages) throws IllegalStateException {
for (BulletinBoardMessage bulletinBoardMessage : bulletinBoardMessages){
digestAndUpdate(bulletinBoardMessage);
}
}
@Override
public long getChecksum() {
return checksum;
}
@Override
public long getChecksum(MessageID messageID) {
update(messageID);
return getChecksum();
}
@Override
public long getChecksum(ByteString messageID) {
update(messageID);
return getChecksum();
}
@Override
public long getChecksum(byte[] messageID) {
update(messageID);
return getChecksum();
}
@Override
public long getChecksum(Collection<MessageID> messageIDs) {
update(messageIDs);
return getChecksum();
}
@Override
public long digestAndGetChecksum(BulletinBoardMessage bulletinBoardMessage) throws IllegalStateException {
digestAndUpdate(bulletinBoardMessage);
return getChecksum();
}
@Override
public long digestAndGetChecksum(Collection<BulletinBoardMessage> bulletinBoardMessages) throws IllegalStateException {
digestAndUpdate(bulletinBoardMessages);
return getChecksum();
}
}

98
meerkat-common/src/main/java/meerkat/comm/MessageInputStream.java Normal file
View File

@ -0,0 +1,98 @@
package meerkat.comm;
import com.google.protobuf.Message;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 21-Feb-16.
* A input stream of Protobuf messages
*/
public class MessageInputStream<T extends Message> implements Iterable<T>{
private T.Builder builder;
private InputStream in;
MessageInputStream(InputStream in, Class<T> type) throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
this.in = in;
this.builder = (T.Builder) type.getMethod("newBuilder").invoke(type);
}
@Override
public Iterator<T> iterator() {
return new Iterator<T>() {
@Override
public boolean hasNext() {
try {
return isAvailable();
} catch (IOException e) {
return false;
}
}
@Override
public T next() {
try {
return readMessage();
} catch (IOException e) {
return null;
}
}
};
}
/**
* Factory class for actually creating a MessageInputStream
*/
public static class MessageInputStreamFactory {
public static <T extends Message> MessageInputStream<T> createMessageInputStream(InputStream in, Class<T> type)
throws IOException, NoSuchMethodException, IllegalAccessException, InvocationTargetException {
return new MessageInputStream<>(in, type);
}
}
public T readMessage() throws IOException{
builder.clear();
builder.mergeDelimitedFrom(in);
return (T) builder.build();
}
public boolean isAvailable() throws IOException {
return (in.available() > 0);
}
public List<T> asList() throws IOException{
List<T> list = new LinkedList<>();
while (isAvailable()){
list.add(readMessage());
}
return list;
}
public void close() throws IOException {
in.close();
}
}

28
meerkat-common/src/main/java/meerkat/comm/MessageOutputStream.java Normal file
View File

@ -0,0 +1,28 @@
package meerkat.comm;
import com.google.protobuf.Message;
import java.io.IOException;
import java.io.OutputStream;
/**
* Created by Arbel Deutsch Peled on 21-Feb-16.
* An output stream of Protobuf messages
*/
public class MessageOutputStream<T extends Message> {
private OutputStream out;
public MessageOutputStream(OutputStream out) throws IOException {
this.out = out;
}
public void writeMessage(T message) throws IOException {
message.writeDelimitedTo(out);
}
public void close() throws IOException {
out.close();
}
}

7
meerkat-common/src/main/java/meerkat/comm/Timestamp.java
View File

@ -1,7 +0,0 @@
package meerkat.comm;
/**
* Created by talm on 24/10/15.
*/
public class Timestamp {
}

13
meerkat-common/src/main/java/meerkat/crypto/Digest.java
View File

@ -1,6 +1,7 @@
package meerkat.crypto;
import com.google.protobuf.Message;
import meerkat.protobuf.BulletinBoardAPI.MessageID;
import java.security.MessageDigest;
@ -13,7 +14,13 @@ public interface Digest {
* (copied from {@link MessageDigest#digest()})
* @return
*/
byte[] digest();
public byte[] digest();
/**
* Completes the hash computation and returns a MessageID Protobuf as output
* @return
*/
public MessageID digestAsMessageID();
/**
* Updates the digest using the specified message (in serialized wire form)
@ -22,12 +29,12 @@ public interface Digest {
* @param msg
* @return
*/
void update(Message msg);
public void update(Message msg);
/**
* Resets the digest for further use.
*/
void reset();
public void reset();
/**
* Clone the current digest state

22
meerkat-common/src/main/java/meerkat/crypto/DigitalSignature.java
View File

@ -5,11 +5,13 @@ import com.google.protobuf.Message;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.KeyStore;
import java.security.SignatureException;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.security.SignatureException;
import java.security.InvalidKeyException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.UnrecoverableKeyException;
import static meerkat.protobuf.Crypto.*;
/**
@ -71,6 +73,20 @@ public interface DigitalSignature {
*/
public boolean verify();
/**
* Load a keystore from an input stream in PKCS12 format.
*
* @param keyStream
* @param password
* @return
* @throws IOException
* @throws CertificateException
* @throws KeyStoreException
* @throws NoSuchAlgorithmException
*/
public KeyStore.Builder getPKCS12KeyStoreBuilder(InputStream keyStream, char[] password)
throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException;
/**
* Loads a private signing key. The keystore must include both the public and private
* key parts.

6
meerkat-common/src/main/java/meerkat/crypto/concrete/ECDSASignature.java
View File

@ -30,7 +30,10 @@ import javax.security.auth.callback.UnsupportedCallbackException;
*
* This class is not thread-safe (each thread should have its own instance).
*/
public class ECDSASignature extends GlobalCryptoSetup implements DigitalSignature {
public class ECDSASignature implements DigitalSignature {
private static GlobalCryptoSetup globalCryptoSetup = GlobalCryptoSetup.getInstance();
final Logger logger = LoggerFactory.getLogger(getClass());
final public static String KEYSTORE_TYPE = "PKCS12";
@ -208,6 +211,7 @@ public class ECDSASignature extends GlobalCryptoSetup implements DigitalSignatur
* @throws KeyStoreException
* @throws NoSuchAlgorithmException
*/
@Override
public KeyStore.Builder getPKCS12KeyStoreBuilder(InputStream keyStream, char[] password)
throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException {
KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE);

5
meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java
View File

@ -31,7 +31,10 @@ import java.util.Random;
/**
* Created by talm on 17/11/15.
*/
public class ECElGamalEncryption extends GlobalCryptoSetup implements Encryption {
public class ECElGamalEncryption implements Encryption {
private static GlobalCryptoSetup globalCryptoSetup = GlobalCryptoSetup.getInstance();
final Logger logger = LoggerFactory.getLogger(getClass());
public final static String KEY_ALGORITHM = "ECDH";

28
meerkat-common/src/main/java/meerkat/crypto/concrete/GlobalCryptoSetup.java
View File

@ -10,13 +10,26 @@ import java.security.Security;
/**
* A class that performs required crypto setup
*/
public class GlobalCryptoSetup {
public final class GlobalCryptoSetup {
private static GlobalCryptoSetup globalCryptoSetup;
final static Logger logger = LoggerFactory.getLogger(GlobalCryptoSetup.class);
static boolean loadedBouncyCastle = false;
static Provider bouncyCastleProvider;
private boolean loadedBouncyCastle = false;
private Provider bouncyCastleProvider;
public static boolean hasSecp256k1Curve() {
private GlobalCryptoSetup() { doSetup(); }
public static GlobalCryptoSetup getInstance() {
if (globalCryptoSetup == null) {
globalCryptoSetup = new GlobalCryptoSetup();
}
return globalCryptoSetup;
}
public boolean hasSecp256k1Curve() {
// For now we just check if the java version is at least 8
String[] version = System.getProperty("java.version").split("\\.");
int major = Integer.parseInt(version[0]);
@ -24,9 +37,11 @@ public class GlobalCryptoSetup {
return ((major > 1) || ((major > 0) && (minor > 7)));
}
public static Provider getBouncyCastleProvider() { doSetup(); return bouncyCastleProvider; }
public Provider getBouncyCastleProvider() {
return bouncyCastleProvider;
}
public static synchronized void doSetup() {
public void doSetup() {
if (bouncyCastleProvider == null) {
bouncyCastleProvider = new BouncyCastleProvider();
// Make bouncycastle our default provider if we're running on a JVM version < 8
@ -39,5 +54,4 @@ public class GlobalCryptoSetup {
}
}
public GlobalCryptoSetup() { doSetup(); }
}

8
meerkat-common/src/main/java/meerkat/crypto/concrete/SHA256Digest.java
View File

@ -3,6 +3,7 @@ package meerkat.crypto.concrete;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
import meerkat.crypto.Digest;
import meerkat.protobuf.BulletinBoardAPI.MessageID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -13,7 +14,7 @@ import java.security.NoSuchAlgorithmException;
/**
* Created by talm on 11/9/15.
*/
public class SHA256Digest extends GlobalCryptoSetup implements Digest {
public class SHA256Digest implements Digest {
final Logger logger = LoggerFactory.getLogger(getClass());
public static final String SHA256 = "SHA-256";
@ -60,6 +61,11 @@ public class SHA256Digest extends GlobalCryptoSetup implements Digest {
return hash.digest();
}
@Override
public MessageID digestAsMessageID() {
return MessageID.newBuilder().setID(ByteString.copyFrom(digest())).build();
}
@Override
public void update(Message msg) {

98
meerkat-common/src/main/java/meerkat/util/BulletinBoardMessageGenerator.java Normal file
View File

@ -0,0 +1,98 @@
package meerkat.util;
import com.google.protobuf.ByteString;
import meerkat.crypto.DigitalSignature;
import meerkat.protobuf.BulletinBoardAPI.*;
import com.google.protobuf.Timestamp;
import java.math.BigInteger;
import java.security.SignatureException;
import java.util.Arrays;
import java.util.Random;
/**
* Created by Arbel Deutsch Peled on 21-Feb-16.
* This class contains methods used to generate random Bulletin Board Messages
*/
public class BulletinBoardMessageGenerator {
private Random random;
public BulletinBoardMessageGenerator(Random random) {
this.random = random;
}
private byte randomByte(){
return (byte) random.nextInt();
}
private String randomString(){
return new BigInteger(130, random).toString(32);
}
/**
* Generates a complete instance of a BulletinBoardMessage
* @param signers contains the (possibly multiple) credentials required to sign the message
* @param timestamp contains the time used in the message
* @param dataSize is the length of the data contained in the message
* @param tagNumber is the number of tags to generate
* @return a random, signed Bulletin Board Message containing random data and tags and the given timestamp
*/
public BulletinBoardMessage generateRandomMessage(DigitalSignature[] signers, Timestamp timestamp, int dataSize, int tagNumber)
throws SignatureException {
// Generate random data.
byte[] data = new byte[dataSize];
String[] tags = new String[tagNumber];
for (int i = 0; i < dataSize; i++) {
data[i] = randomByte();
}
for (int i = 0; i < tagNumber; i++) {
tags[i] = randomString();
}
UnsignedBulletinBoardMessage unsignedMessage =
UnsignedBulletinBoardMessage.newBuilder()
.setData(ByteString.copyFrom(data))
.setTimestamp(timestamp)
.addAllTag(Arrays.asList(tags))
.build();
BulletinBoardMessage.Builder messageBuilder =
BulletinBoardMessage.newBuilder()
.setMsg(unsignedMessage);
for (int i = 0 ; i < signers.length ; i++) {
signers[i].updateContent(unsignedMessage);
messageBuilder.addSig(signers[i].sign());
}
return messageBuilder.build();
}
/**
* Generates a complete instance of a BulletinBoardMessage
* @param signers contains the (possibly multiple) credentials required to sign the message
* @param dataSize is the length of the data contained in the message
* @param tagNumber is the number of tags to generate
* @return a random, signed Bulletin Board Message containing random data, tags and timestamp
*/
public BulletinBoardMessage generateRandomMessage(DigitalSignature[] signers, int dataSize, int tagNumber)
throws SignatureException {
Timestamp timestamp = Timestamp.newBuilder()
.setSeconds(random.nextLong())
.setNanos(random.nextInt())
.build();
return generateRandomMessage(signers, timestamp, dataSize, tagNumber);
}
}

107
meerkat-common/src/main/java/meerkat/util/BulletinBoardUtils.java Normal file
View File

@ -0,0 +1,107 @@
package meerkat.util;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.LinkedList;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 16-Feb-16.
*/
public class BulletinBoardUtils {
/**
* Searches the tags in the message for one that begins with given prefix
* @param message is the message to search
* @param prefix is the given prefix
* @return the tag without the prefix, if found, or null if not found
*/
public static String findTagWithPrefix(BulletinBoardMessage message, String prefix) {
for (String tag : message.getMsg().getTagList()){
if (tag.startsWith(prefix)) {
return tag.substring(prefix.length());
}
}
return null;
}
/**
* Searches the tags in a message for tags that do not contain a given list of prefixes
* @param message is the message to search
* @param prefixes is the list of prefixes
* @return a list of the tags that do *not* contain any of the given prefixes
*/
public static List<String> removePrefixTags(BulletinBoardMessage message, Iterable<String> prefixes) {
if (prefixes == null)
return message.getMsg().getTagList();
List<String> result = new LinkedList<>();
for (String tag : message.getMsg().getTagList()){
boolean found = false;
for (String prefix : prefixes){
if (tag.startsWith(prefix)){
found = true;
break;
}
}
if (!found) {
result.add(tag);
}
}
return result;
}
/**
* This method creates a Timestamp Protobuf from a time specification
* @param timeInMillis is the time to encode since the Epoch time in milliseconds
* @return a Timestamp Protobuf encoding of the given time
*/
public static com.google.protobuf.Timestamp toTimestampProto(long timeInMillis) {
return com.google.protobuf.Timestamp.newBuilder()
.setSeconds(timeInMillis / 1000)
.setNanos((int) ((timeInMillis % 1000) * 1000000))
.build();
}
/**
* This method creates a Timestamp Protobuf from the current system time
* @return a Timestamp Protobuf encoding of the current system time
*/
public static com.google.protobuf.Timestamp getCurrentTimestampProto() {
return toTimestampProto(System.currentTimeMillis());
}
/**
* This method converts an SQL Timestamp object into a Protobuf Timestamp object
* @param sqlTimestamp is the SQL Timestamp
* @return an equivalent Protobuf Timestamp
*/
public static com.google.protobuf.Timestamp toTimestampProto(java.sql.Timestamp sqlTimestamp) {
return toTimestampProto(sqlTimestamp.getTime());
}
/**
* This method converts a Protobuf Timestamp object into an SQL Timestamp object
* @param protoTimestamp is the Protobuf Timestamp
* @return an equivalent SQL Timestamp
*/
public static java.sql.Timestamp toSQLTimestamp(com.google.protobuf.Timestamp protoTimestamp) {
return new java.sql.Timestamp(protoTimestamp.getSeconds() * 1000 + protoTimestamp.getNanos() / 1000000);
}
}

30
meerkat-common/src/main/java/meerkat/util/TimeStampComparator.java Normal file
View File

@ -0,0 +1,30 @@
package meerkat.util;
import com.google.protobuf.Timestamp;
import java.util.Comparator;
/**
* Created by Arbel Deutsch Peled on 20-Feb-16.
*/
public class TimestampComparator implements Comparator<com.google.protobuf.Timestamp> {
@Override
public int compare(Timestamp o1, Timestamp o2) {
if (o1.getSeconds() != o2.getSeconds()){
return o1.getSeconds() > o2.getSeconds() ? 2 : -2;
} else if (o1.getNanos() != o2.getNanos()){
return o1.getNanos() > o2.getNanos() ? 1 : -1;
} else{
return 0;
}
}
}

89
meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto
View File

@ -5,6 +5,7 @@ package meerkat;
option java_package = "meerkat.protobuf";
import 'meerkat/crypto.proto';
import 'google/protobuf/timestamp.proto';
message BoolMsg {
bool value = 1;
@ -21,11 +22,14 @@ message MessageID {
}
message UnsignedBulletinBoardMessage {
// Optional tags describing message
// Optional tags describing message; Used for message retrieval
repeated string tag = 1;
// Timestamp of the message (as defined by client)
google.protobuf.Timestamp timestamp = 2;
// The actual content of the message
bytes data = 2;
bytes data = 3;
}
message BulletinBoardMessage {
@ -38,6 +42,7 @@ message BulletinBoardMessage {
// Signature of message (and tags), excluding the entry number.
repeated meerkat.Signature sig = 3;
}
message BulletinBoardMessageList {
@ -50,13 +55,16 @@ enum FilterType {
MSG_ID = 0; // Match exact message ID
EXACT_ENTRY = 1; // Match exact entry number in database (chronological)
MAX_ENTRY = 2; // Find all entries in database up to specified entry number (chronological)
SIGNER_ID = 3; // Find all entries in database that correspond to specific signature (signer)
TAG = 4; // Find all entries in database that have a specific tag
MIN_ENTRY = 3; // Find all entries in database starting from specified entry number (chronological)
SIGNER_ID = 4; // Find all entries in database that correspond to specific signature (signer)
TAG = 5; // Find all entries in database that have a specific tag
AFTER_TIME = 6; // Find all entries in database that occurred on or after a given timestamp
BEFORE_TIME = 7; // Find all entries in database that occurred on or before a given timestamp
// NOTE: The MAX_MESSAGES filter must remain the last filter type
// This is because the condition it specifies in an SQL statement must come last in the statement
// Keeping it last here allows for easily sorting the filters and keeping the code general
MAX_MESSAGES = 5; // Return at most some specified number of messages
MAX_MESSAGES = 8; // Return at most some specified number of messages
}
message MessageFilter {
@ -68,6 +76,7 @@ message MessageFilter {
int64 entry = 3;
string tag = 4;
int64 maxMessages = 5;
google.protobuf.Timestamp timestamp = 6;
}
}
@ -77,4 +86,74 @@ message MessageFilterList {
// To be implemented using intersection ("AND") operations.
repeated MessageFilter filter = 1;
}
// This message is used to start a batch transfer to the Bulletin Board Server
message BeginBatchMessage {
bytes signerId = 1; // Unique signer identifier
int32 batchId = 2; // Unique identifier for the batch (unique per signer)
repeated string tag = 3; // Tags for the batch message
}
// This message is used to finalize and sign a batch transfer to the Bulletin Board Server
message CloseBatchMessage {
int32 batchId = 1; // Unique identifier for the batch (unique per signer)
int32 batchLength = 2; // Number of messages in the batch
google.protobuf.Timestamp timestamp = 3; // Timestamp of the batch (as defined by client)
meerkat.Signature sig = 4; // Signature on the (ordered) batch messages
}
// Container for single batch message data
message BatchData {
bytes data = 1;
}
// List of BatchData; Only used for testing
message BatchDataList {
repeated BatchData data = 1;
}
// These messages comprise a batch message
message BatchMessage {
bytes signerId = 1; // Unique signer identifier
int32 batchId = 2; // Unique identifier for the batch (unique per signer)
int32 serialNum = 3; // Location of the message in the batch: starting from 0
BatchData data = 4; // Actual data
}
// This message defines which batch to read and from which location to start reading
message BatchSpecificationMessage {
bytes signerId = 1; // Unique signer identifier
int32 batchId = 2; // Unique identifier for the batch (unique per signer)
int32 startPosition = 3; // Position in batch to start reading from
}
// This message is used to define a single query to the server to ascertain whether or not the server is synched with the client
// up till a specified timestamp
message SingleSyncQuery {
google.protobuf.Timestamp timeOfSync = 1;
int64 checksum = 2;
}
// This message defines a complete server sync query
message SyncQuery {
MessageFilterList filterList = 1;
repeated SingleSyncQuery query = 2;
}
// This message defines the server's response format to a sync query
message SyncQueryResponse {
// Serial entry number of current last entry in database
// Set to zero (0) in case no query checksums match
int64 lastEntryNum = 1;
// Largest value of timestamp for which the checksums match
google.protobuf.Timestamp lastTimeOfSync = 2;
}

98
meerkat-common/src/test/java/meerkat/comm/MessageStreamTest.java Normal file
View File

@ -0,0 +1,98 @@
package meerkat.comm;
import com.google.protobuf.*;
import meerkat.comm.MessageInputStream.MessageInputStreamFactory;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto;
import meerkat.util.BulletinBoardMessageComparator;
import org.junit.Test;
import java.io.*;
import java.lang.reflect.InvocationTargetException;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
import static org.hamcrest.core.IsEqual.equalTo;
/**
* Created by Arbel Deutsch Peled on 21-Feb-16.
* Tests for MessageInputStream and MessageOutputStream classes
*/
public class MessageStreamTest {
@Test
public void testWithBulletinBoardMessages() {
MessageOutputStream<BulletinBoardMessage> out;
MessageInputStream<BulletinBoardMessage> in;
ByteArrayOutputStream stream = new ByteArrayOutputStream();
BulletinBoardMessageComparator comparator = new BulletinBoardMessageComparator();
byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4};
byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14};
byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24};
try {
out = new MessageOutputStream<>(stream);
} catch (IOException e) {
System.err.println(e.getMessage());
assertThat("Error creating streams: " + e.getMessage(), false);
return;
}
BulletinBoardMessage message = BulletinBoardMessage.newBuilder()
.setEntryNum(1)
.setMsg(UnsignedBulletinBoardMessage.newBuilder()
.setData(ByteString.copyFrom(b1))
.addTag("Test")
.addTag("1234")
.setTimestamp(com.google.protobuf.Timestamp.newBuilder()
.setSeconds(19823451)
.setNanos(2134)
.build())
.build())
.addSig(Crypto.Signature.newBuilder()
.setSignerId(ByteString.copyFrom(b2))
.setData(ByteString.copyFrom(b3))
.build())
.build();
try {
out.writeMessage(message);
} catch (IOException e) {
System.err.println(e.getMessage());
assertThat("Error writing message: " + e.getMessage(), false);
}
try {
in = MessageInputStreamFactory.createMessageInputStream(
new ByteArrayInputStream(stream.toByteArray()),
BulletinBoardMessage.class);
assertThat("Retrieved message was not identical to send message", comparator.compare(message, in.readMessage()), is(equalTo(0)));
} catch (IOException e) {
System.err.println(e.getMessage());
assertThat("Error reading message: " + e.getMessage(), false);
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
System.err.println(e.getMessage());
assertThat("Error creating input stream " + e.getMessage(), false);
}
}
}

5
meerkat-common/src/test/java/meerkat/crypto/concrete/ECElGamalUtils.java
View File

@ -26,6 +26,9 @@ import java.security.spec.InvalidKeySpecException;
* utilities for ECElgamal
*/
public class ECElGamalUtils {
private static GlobalCryptoSetup globalCryptoSetup = GlobalCryptoSetup.getInstance();
final static Logger logger = LoggerFactory.getLogger(ECElGamalUtils.class);
public final static String ENCRYPTION_KEY_ALGORITHM = "ECDH";
@ -43,7 +46,7 @@ public class ECElGamalUtils {
try {
KeyFactory fact = KeyFactory.getInstance(ENCRYPTION_KEY_ALGORITHM,
GlobalCryptoSetup.getBouncyCastleProvider());
globalCryptoSetup.getBouncyCastleProvider());
PublicKey javaPk = fact.generatePublic(pubKeySpec);
ConcreteCrypto.ElGamalPublicKey serializedPk = ConcreteCrypto.ElGamalPublicKey.newBuilder()
.setSubjectPublicKeyInfo(ByteString.copyFrom(javaPk.getEncoded())).build();

4
restful-api-common/src/main/java/meerkat/rest/Constants.java
View File

@ -5,8 +5,4 @@ package meerkat.rest;
*/
public interface Constants {
public static final String MEDIATYPE_PROTOBUF = "application/x-protobuf";
public static final String BULLETIN_BOARD_SERVER_PATH = "/bbserver";
public static final String READ_MESSAGES_PATH = "/readmessages";
public static final String POST_MESSAGE_PATH = "/postmessage";
}