meerkat
/
meerkat-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into dist_dec 

# Conflicts:
#	meerkat-common/src/main/java/meerkat/crypto/concrete/ECElGamalEncryption.java
#	meerkat-common/src/main/proto/meerkat/BulletinBoardAPI.proto
#	settings.gradle
mixer
Hai Brenner 2016-08-09 14:30:40 +03:00
parent e677355040 ce40a04ac7
commit 6b512e078b
219 changed files with 17984 additions and 1205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -13,4 +13,6 @@ out
*.prefs
*.project
*.classpath
bulletin-board-server/local-instances/meerkat.db
*.db
*.sql
.arcconfig

242
bulletin-board-client/build.gradle Normal file
View File

@ -0,0 +1,242 @@
plugins {
id "us.kirchmeier.capsule" version "1.0.1"
id 'com.google.protobuf' version '0.7.0'
}
apply plugin: 'java'
apply plugin: 'com.google.protobuf'
apply plugin: 'eclipse'
apply plugin: 'idea'
apply plugin: 'maven-publish'
// Is this a snapshot version?
ext { isSnapshot = false }
ext {
groupId = 'org.factcenter.meerkat'
nexusRepository = "https://cs.idc.ac.il/nexus/content/groups/${isSnapshot ? 'unstable' : 'public'}/"
// Credentials for IDC nexus repositories (needed only for using unstable repositories and publishing)
// Should be set in ${HOME}/.gradle/gradle.properties
nexusUser = project.hasProperty('nexusUser') ? project.property('nexusUser') : ""
nexusPassword = project.hasProperty('nexusPassword') ? project.property('nexusPassword') : ""
}
description = "Meerkat Voting Common Library"
// Your project version
version = "0.0"
version += "${isSnapshot ? '-SNAPSHOT' : ''}"
dependencies {
// Meerkat common
compile project(':meerkat-common')
compile project(':restful-api-common')
// Jersey for RESTful API
compile 'org.glassfish.jersey.containers:jersey-container-servlet:2.22.+'
compile 'org.xerial:sqlite-jdbc:3.7.+'
// Logging
compile 'org.slf4j:slf4j-api:1.7.7'
runtime 'ch.qos.logback:logback-classic:1.1.2'
runtime 'ch.qos.logback:logback-core:1.1.2'
// Google protobufs
compile 'com.google.protobuf:protobuf-java:3.+'
// Crypto
compile 'org.factcenter.qilin:qilin:1.1+'
compile 'org.bouncycastle:bcprov-jdk15on:1.53'
compile 'org.bouncycastle:bcpkix-jdk15on:1.53'
// Depend on test resources from meerkat-common
testCompile project(path: ':meerkat-common', configuration: 'testOutput')
// Depend on server compilation for the non-integration tests
testCompile project(path: ':bulletin-board-server')
testCompile 'junit:junit:4.+'
testCompile 'org.hamcrest:hamcrest-all:1.3'
runtime 'org.codehaus.groovy:groovy:2.4.+'
}
test {
exclude '**/*IntegrationTest*'
// outputs.upToDateWhen { false }
}
task integrationTest(type: Test) {
include '**/*IntegrationTest*'
// debug = true
outputs.upToDateWhen { false }
}
/*==== You probably don't have to edit below this line =======*/
// Setup test configuration that can appear as a dependency in
// other subprojects
configurations {
testOutput.extendsFrom (testCompile)
}
task testJar(type: Jar, dependsOn: testClasses) {
classifier = 'tests'
from sourceSets.test.output
}
artifacts {
testOutput testJar
}
// The run task added by the application plugin
// is also of type JavaExec.
tasks.withType(JavaExec) {
// Assign all Java system properties from
// the command line to the JavaExec task.
systemProperties System.properties
}
protobuf {
// Configure the protoc executable
protoc {
// Download from repositories
artifact = 'com.google.protobuf:protoc:3.+'
}
}
idea {
module {
project.sourceSets.each { sourceSet ->
def srcDir = "${protobuf.generatedFilesBaseDir}/$sourceSet.name/java"
// add protobuf generated sources to generated source dir.
if ("test".equals(sourceSet.name)) {
testSourceDirs += file(srcDir)
} else {
sourceDirs += file(srcDir)
}
generatedSourceDirs += file(srcDir)
}
// Don't exclude build directory
excludeDirs -= file(buildDir)
}
}
/*===================================
* "Fat" Build targets
*===================================*/
if (project.hasProperty('mainClassName') && (mainClassName != null)) {
task mavenCapsule(type: MavenCapsule) {
description = "Generate a capsule jar that automatically downloads and caches dependencies when run."
applicationClass mainClassName
destinationDir = buildDir
}
task fatCapsule(type: FatCapsule) {
description = "Generate a single capsule jar containing everything. Use -Pfatmain=... to override main class"
destinationDir = buildDir
def fatMain = hasProperty('fatmain') ? fatmain : mainClassName
applicationClass fatMain
def testJar = hasProperty('test')
if (hasProperty('fatmain')) {
appendix = "fat-${fatMain}"
} else {
appendix = "fat"
}
if (testJar) {
from sourceSets.test.output
}
}
}
/*===================================
* Repositories
*===================================*/
repositories {
mavenLocal();
// Prefer the local nexus repository (it may have 3rd party artifacts not found in mavenCentral)
maven {
url nexusRepository
if (isSnapshot) {
credentials { username
password
username nexusUser
password nexusPassword
}
}
}
// Use 'maven central' for other dependencies.
mavenCentral()
}
task "info" << {
println "Project: ${project.name}"
println "Description: ${project.description}"
println "--------------------------"
println "GroupId: $groupId"
println "Version: $version (${isSnapshot ? 'snapshot' : 'release'})"
println ""
}
info.description 'Print some information about project parameters'
/*===================================
* Publishing
*===================================*/
publishing {
publications {
mavenJava(MavenPublication) {
groupId project.groupId
pom.withXml {
asNode().appendNode('description', project.description)
}
from project.components.java
}
}
repositories {
maven {
url "https://cs.idc.ac.il/nexus/content/repositories/${project.isSnapshot ? 'snapshots' : 'releases'}"
credentials { username
password
username nexusUser
password nexusPassword
}
}
}
}

23
bulletin-board-client/src/main/java/meerkat/bulletinboard/BatchDataContainer.java Normal file
View File

@ -0,0 +1,23 @@
package meerkat.bulletinboard;
import meerkat.protobuf.BulletinBoardAPI.BatchChunk;
import meerkat.bulletinboard.AsyncBulletinBoardClient.BatchIdentifier;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 17-Jan-16.
* Used to store the complete data required for sending a batch data list inside a single object
*/
public class BatchDataContainer {
public final MultiServerBatchIdentifier batchId;
public final List<BatchChunk> batchChunkList;
public final int startPosition;
public BatchDataContainer(MultiServerBatchIdentifier batchId, List<BatchChunk> batchChunkList, int startPosition) {
this.batchId = batchId;
this.batchChunkList = batchChunkList;
this.startPosition = startPosition;
}
}

82
bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJob.java
View File

@ -1,82 +0,0 @@
package meerkat.bulletinboard;
import com.google.protobuf.Message;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 09-Dec-15.
*
* This class specifies the job that is required of a Bulletin Board Client Worker
*/
public class BulletinClientJob {
public static enum JobType{
POST_MESSAGE, // Post a message to servers
READ_MESSAGES, // Read messages according to some given filter (any server will do)
GET_REDUNDANCY // Check the redundancy of a specific message in the databases
}
private List<String> serverAddresses;
private int minServers; // The minimal number of servers the job must be successful on for the job to be completed
private final JobType jobType;
private final Message payload; // The information associated with the job type
private int maxRetry; // Number of retries for this job; set to -1 for infinite retries
public BulletinClientJob(List<String> serverAddresses, int minServers, JobType jobType, Message payload, int maxRetry) {
this.serverAddresses = serverAddresses;
this.minServers = minServers;
this.jobType = jobType;
this.payload = payload;
this.maxRetry = maxRetry;
}
public void updateServerAddresses(List<String> newServerAdresses) {
this.serverAddresses = newServerAdresses;
}
public List<String> getServerAddresses() {
return serverAddresses;
}
public int getMinServers() {
return minServers;
}
public JobType getJobType() {
return jobType;
}
public Message getPayload() {
return payload;
}
public int getMaxRetry() {
return maxRetry;
}
public void shuffleAddresses() {
Collections.shuffle(serverAddresses);
}
public void decMinServers(){
minServers--;
}
public void decMaxRetry(){
if (maxRetry > 0) {
maxRetry--;
}
}
public boolean isRetry(){
return (maxRetry != 0);
}
}

29
bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientJobResult.java
View File

@ -1,29 +0,0 @@
package meerkat.bulletinboard;
import com.google.protobuf.Message;
/**
* Created by Arbel Deutsch Peled on 09-Dec-15.
*
* This class contains the end status and result of a Bulletin Board Client Job.
*/
public final class BulletinClientJobResult {
private final BulletinClientJob job; // Stores the job the result refers to
private final Message result; // The result of the job; valid only if success==true
public BulletinClientJobResult(BulletinClientJob job, Message result) {
this.job = job;
this.result = result;
}
public BulletinClientJob getJob() {
return job;
}
public Message getResult() {
return result;
}
}

223
bulletin-board-client/src/main/java/meerkat/bulletinboard/BulletinClientWorker.java
View File

@ -1,217 +1,38 @@
package meerkat.bulletinboard;
import com.google.protobuf.Message;
import meerkat.comm.CommunicationException;
import meerkat.crypto.Digest;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.rest.Constants;
import meerkat.rest.ProtobufMessageBodyReader;
import meerkat.rest.ProtobufMessageBodyWriter;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Callable;
/**
* Created by Arbel Deutsch Peled on 09-Dec-15.
*
* This class implements the actual communication with the Bulletin Board Servers.
* This class handles bulletin client work.
* It is meant to be used in a multi-threaded environment.
*/
//TODO: Maybe make this abstract and inherit from it.
public class BulletinClientWorker implements Callable<BulletinClientJobResult> {
public abstract class BulletinClientWorker<IN> {
private final BulletinClientJob job; // The requested job to be handled
protected final IN payload; // Payload of the job
public BulletinClientWorker(BulletinClientJob job){
this.job = job;
private int maxRetry; // Number of retries for this job; set to -1 for infinite retries
public BulletinClientWorker(IN payload, int maxRetry) {
this.payload = payload;
this.maxRetry = maxRetry;
}
// This resource enabled creation of a single Client per thread.
private static final ThreadLocal<Client> clientLocal =
new ThreadLocal<Client> () {
@Override protected Client initialValue() {
Client client;
client = ClientBuilder.newClient();
client.register(ProtobufMessageBodyReader.class);
client.register(ProtobufMessageBodyWriter.class);
public IN getPayload() {
return payload;
}
return client;
}
};
// This resource enables creation of a single Digest per thread.
private static final ThreadLocal<Digest> digestLocal =
new ThreadLocal<Digest> () {
@Override protected Digest initialValue() {
Digest digest;
digest = new SHA256Digest(); //TODO: Make this generic.
return digest;
}
};
/**
* This method carries out the actual communication with the servers via HTTP Post
* It accesses the servers according to the job it received and updates said job as it goes
* The method will only iterate once through the server list, removing servers from the list when they are no longer required
* In a POST_MESSAGE job: successful post to a server results in removing the server from the list
* In a GET_REDUNDANCY job: no server is removed from the list and the (absolute) number of servers in which the message was found is returned
* In a READ_MESSAGES job: successful retrieval from any server terminates the method and returns the received values; The list is not changed
* @return The original job, modified to fit the current state and the required output (if any) of the operation
* @throws IllegalArgumentException
* @throws CommunicationException
*/
public BulletinClientJobResult call() throws IllegalArgumentException, CommunicationException{
Client client = clientLocal.get();
Digest digest = digestLocal.get();
WebTarget webTarget;
Response response;
String requestPath;
Message msg;
List<String> serverAddresses = new LinkedList<String>(job.getServerAddresses());
Message payload = job.getPayload();
BulletinBoardMessageList msgList;
int count = 0; // Used to count number of servers which contain the required message in a GET_REDUNDANCY request.
job.shuffleAddresses(); // This is done to randomize the order of access to servers primarily for READ operations
// Prepare the request.
switch(job.getJobType()) {
case POST_MESSAGE:
// Make sure the payload is a BulletinBoardMessage
if (!(payload instanceof BulletinBoardMessage)) {
throw new IllegalArgumentException("Cannot post an object that is not an instance of BulletinBoardMessage");
}
msg = payload;
requestPath = Constants.POST_MESSAGE_PATH;
break;
case READ_MESSAGES:
// Make sure the payload is a MessageFilterList
if (!(payload instanceof MessageFilterList)) {
throw new IllegalArgumentException("Read failed: an instance of MessageFilterList is required as payload for a READ_MESSAGES operation");
}
msg = payload;
requestPath = Constants.READ_MESSAGES_PATH;
break;
case GET_REDUNDANCY:
// Make sure the payload is a MessageId
if (!(payload instanceof MessageID)) {
throw new IllegalArgumentException("Cannot search for an object that is not an instance of MessageID");
}
requestPath = Constants.READ_MESSAGES_PATH;
msg = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(((MessageID) payload).getID())
.build()
).build();
break;
default:
throw new IllegalArgumentException("Unsupported job type");
}
// Iterate through servers
Iterator<String> addressIterator = serverAddresses.iterator();
while (addressIterator.hasNext()) {
// Send request to Server
String address = addressIterator.next();
webTarget = client.target(address).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(requestPath);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF));
// Retrieve answer
switch(job.getJobType()) {
case POST_MESSAGE:
try {
response.readEntity(BoolMsg.class); // If a BoolMsg entity is returned: the post was successful
addressIterator.remove(); // Post to this server succeeded: remove server from list
job.decMinServers();
} catch (ProcessingException | IllegalStateException e) {} // Post to this server failed: retry next time
finally {
response.close();
}
break;
case GET_REDUNDANCY:
try {
msgList = response.readEntity(BulletinBoardMessageList.class); // If a BulletinBoardMessageList is returned: the read was successful
if (msgList.getMessageList().size() > 0){ // Message was found in the server.
count++;
}
} catch (ProcessingException | IllegalStateException e) {} // Read failed: try with next server
finally {
response.close();
}
break;
case READ_MESSAGES:
try {
msgList = response.readEntity(BulletinBoardMessageList.class); // If a BulletinBoardMessageList is returned: the read was successful
return new BulletinClientJobResult(job, msgList); // Return the result
} catch (ProcessingException | IllegalStateException e) {} // Read failed: try with next server
finally {
response.close();
}
break;
}
}
// Return result (if haven't done so yet)
switch(job.getJobType()) {
case POST_MESSAGE:
// The job now contains the information required to ascertain whether enough server posts have succeeded
// It will also contain the list of servers in which the post was not successful
job.updateServerAddresses(serverAddresses);
return new BulletinClientJobResult(job, null);
case GET_REDUNDANCY:
// Return the number of servers in which the message was found
// The job now contains the list of these servers
return new BulletinClientJobResult(job, IntMsg.newBuilder().setValue(count).build());
case READ_MESSAGES:
// A successful operation would have already returned an output
// Therefore: no server access was successful
throw new CommunicationException("Could not access any server");
default: // This is required for successful compilation
throw new IllegalArgumentException("Unsupported job type");
public int getMaxRetry() {
return maxRetry;
}
public void decMaxRetry(){
if (maxRetry > 0) {
maxRetry--;
}
}
public boolean isRetry(){
return (maxRetry != 0);
}
}

490
bulletin-board-client/src/main/java/meerkat/bulletinboard/CachedBulletinBoardClient.java Normal file
View File

@ -0,0 +1,490 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.Timestamp;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto.Signature;
import meerkat.protobuf.Voting.*;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 03-Mar-16.
* This is a full-fledged implementation of a Bulletin Board Client
* It provides asynchronous access to several remote servers, as well as a local cache
* Read operations are performed on the local server
* Batch reads are performed on the local server and, if they fail, also on the remote servers
* Write operations are performed on the local server
* A Synchronizer is employed in order to keep the remote server up to date
* After any read is carried out, a subscription is made for the specific query to make sure the local DB will be updated
* The database also employs a synchronizer which makes sure local data is sent to the remote servers
*/
public class CachedBulletinBoardClient implements SubscriptionBulletinBoardClient {
private final AsyncBulletinBoardClient localClient;
private final AsyncBulletinBoardClient remoteClient;
private final AsyncBulletinBoardClient queueClient;
private final BulletinBoardSubscriber subscriber;
private final BulletinBoardSynchronizer synchronizer;
private Thread syncThread;
private final static int DEFAULT_WAIT_CAP = 3000;
private final static int DEFAULT_SLEEP_INTERVAL = 3000;
private class SubscriptionStoreCallback implements FutureCallback<List<BulletinBoardMessage>> {
private final FutureCallback<List<BulletinBoardMessage>> callback;
public SubscriptionStoreCallback(){
callback = null;
}
public SubscriptionStoreCallback(FutureCallback<List<BulletinBoardMessage>> callback){
this.callback = callback;
}
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
for (BulletinBoardMessage msg : result) {
try {
if (msg.getMsg().getDataTypeCase() == UnsignedBulletinBoardMessage.DataTypeCase.MSGID) {
// This is a batch message: need to upload batch data as well as the message itself
BulletinBoardMessage completeMessage = localClient.readBatchData(msg);
localClient.postMessage(completeMessage);
} else {
// This is a regular message: post it
localClient.postMessage(msg);
}
} catch (CommunicationException ignored) {
// TODO: log
}
}
}
@Override
public void onFailure(Throwable t) {
if (callback != null) {
callback.onFailure(t); // This is some hard error that cannot be dealt with
}
}
}
/**
* Creates a Cached Client
* Assumes all parameters are initialized
* @param localClient is a Client for the local instance
* @param remoteClient is a Client for the remote instance(s); Should have endless retries for post operations
* @param subscriber is a subscription service to the remote instance(s)
* @param queueClient is a client for a local deletable server to be used as a queue for not-yet-uploaded messages
*/
public CachedBulletinBoardClient(AsyncBulletinBoardClient localClient,
AsyncBulletinBoardClient remoteClient,
BulletinBoardSubscriber subscriber,
DeletableSubscriptionBulletinBoardClient queueClient,
int sleepInterval,
int waitCap) {
this.localClient = localClient;
this.remoteClient = remoteClient;
this.subscriber = subscriber;
this.queueClient = queueClient;
this.synchronizer = new SimpleBulletinBoardSynchronizer(sleepInterval,waitCap);
synchronizer.init(queueClient, remoteClient);
syncThread = new Thread(synchronizer);
syncThread.start();
}
/**
* Creates a Cached Client
* Used default values foe the time caps
* */
public CachedBulletinBoardClient(AsyncBulletinBoardClient localClient,
AsyncBulletinBoardClient remoteClient,
BulletinBoardSubscriber subscriber,
DeletableSubscriptionBulletinBoardClient queue) {
this(localClient, remoteClient, subscriber, queue, DEFAULT_SLEEP_INTERVAL, DEFAULT_WAIT_CAP);
}
@Override
public MessageID postMessage(final BulletinBoardMessage msg, final FutureCallback<Boolean> callback) {
return localClient.postMessage(msg, new FutureCallback<Boolean>() {
@Override
public void onSuccess(Boolean result) {
remoteClient.postMessage(msg, callback);
}
@Override
public void onFailure(Throwable t) {
if (callback != null)
callback.onFailure(t);
}
});
}
@Override
public MessageID postAsBatch(final BulletinBoardMessage msg, final int chunkSize, final FutureCallback<Boolean> callback) {
return localClient.postAsBatch(msg, chunkSize, new FutureCallback<Boolean>() {
@Override
public void onSuccess(Boolean result) {
remoteClient.postAsBatch(msg, chunkSize, callback);
}
@Override
public void onFailure(Throwable t) {
if (callback != null)
callback.onFailure(t);
}
});
}
@Override
public void beginBatch(final Iterable<String> tags, final FutureCallback<BatchIdentifier> callback) {
localClient.beginBatch(tags, new FutureCallback<BatchIdentifier>() {
private BatchIdentifier localIdentifier;
@Override
public void onSuccess(BatchIdentifier result) {
localIdentifier = result;
remoteClient.beginBatch(tags, new FutureCallback<BatchIdentifier>() {
@Override
public void onSuccess(BatchIdentifier result) {
if (callback != null)
callback.onSuccess(new CachedClientBatchIdentifier(localIdentifier, result));
}
@Override
public void onFailure(Throwable t) {
if (callback != null)
callback.onFailure(t);
}
});
}
@Override
public void onFailure(Throwable t) {
if (callback != null)
callback.onFailure(t);
}
});
}
@Override
public void postBatchData(final BatchIdentifier batchIdentifier, final List<BatchChunk> batchChunkList,
final int startPosition, final FutureCallback<Boolean> callback) throws IllegalArgumentException{
if (!(batchIdentifier instanceof CachedClientBatchIdentifier)){
throw new IllegalArgumentException("Error: batch identifier supplied was not created by this class.");
}
final CachedClientBatchIdentifier identifier = (CachedClientBatchIdentifier) batchIdentifier;
localClient.postBatchData(identifier.getLocalIdentifier(), batchChunkList, startPosition, new FutureCallback<Boolean>() {
@Override
public void onSuccess(Boolean result) {
remoteClient.postBatchData(identifier.getRemoteIdentifier(), batchChunkList, startPosition, callback);
}
@Override
public void onFailure(Throwable t) {
if (callback != null)
callback.onFailure(t);
}
});
}
@Override
public void postBatchData(final BatchIdentifier batchIdentifier, final List<BatchChunk> batchChunkList, final FutureCallback<Boolean> callback)
throws IllegalArgumentException{
if (!(batchIdentifier instanceof CachedClientBatchIdentifier)){
throw new IllegalArgumentException("Error: batch identifier supplied was not created by this class.");
}
final CachedClientBatchIdentifier identifier = (CachedClientBatchIdentifier) batchIdentifier;
localClient.postBatchData(identifier.getLocalIdentifier(), batchChunkList, new FutureCallback<Boolean>() {
@Override
public void onSuccess(Boolean result) {
remoteClient.postBatchData(identifier.getRemoteIdentifier(), batchChunkList, callback);
}
@Override
public void onFailure(Throwable t) {
if (callback != null)
callback.onFailure(t);
}
});
}
@Override
public void closeBatch(final BatchIdentifier batchIdentifier, final Timestamp timestamp, final Iterable<Signature> signatures,
final FutureCallback<Boolean> callback) {
if (!(batchIdentifier instanceof CachedClientBatchIdentifier)){
throw new IllegalArgumentException("Error: batch identifier supplied was not created by this class.");
}
final CachedClientBatchIdentifier identifier = (CachedClientBatchIdentifier) batchIdentifier;
localClient.closeBatch(identifier.getLocalIdentifier(), timestamp, signatures, new FutureCallback<Boolean>() {
@Override
public void onSuccess(Boolean result) {
remoteClient.closeBatch(identifier.getRemoteIdentifier(), timestamp, signatures, callback);
}
@Override
public void onFailure(Throwable t) {
if (callback != null)
callback.onFailure(t);
}
});
}
@Override
public void getRedundancy(MessageID id, FutureCallback<Float> callback) {
remoteClient.getRedundancy(id, callback);
}
@Override
public void readMessages(MessageFilterList filterList, final FutureCallback<List<BulletinBoardMessage>> callback) {
localClient.readMessages(filterList, callback);
subscriber.subscribe(filterList, new SubscriptionStoreCallback(callback));
}
@Override
public void readMessage(final MessageID msgID, final FutureCallback<BulletinBoardMessage> callback) {
localClient.readMessage(msgID, new FutureCallback<BulletinBoardMessage>() {
@Override
public void onSuccess(BulletinBoardMessage result) {
if (callback != null)
callback.onSuccess(result); // Read from local client was successful
}
@Override
public void onFailure(Throwable t) {
// Read from local unsuccessful: try to read from remote
remoteClient.readMessage(msgID, new FutureCallback<BulletinBoardMessage>() {
@Override
public void onSuccess(BulletinBoardMessage result) {
// Read from remote was successful: store in local and return result
localClient.postMessage(result, null);
if (callback != null)
callback.onSuccess(result);
}
@Override
public void onFailure(Throwable t) {
// Read from remote was unsuccessful: report error
if (callback != null)
callback.onFailure(t);
}
});
}
});
}
@Override
public void readBatchData(final BulletinBoardMessage stub, final FutureCallback<BulletinBoardMessage> callback) throws IllegalArgumentException {
localClient.readBatchData(stub, new FutureCallback<BulletinBoardMessage>() {
@Override
public void onSuccess(BulletinBoardMessage result) {
if (callback != null)
callback.onSuccess(result); // Read from local client was successful
}
@Override
public void onFailure(Throwable t) {
// Read from local unsuccessful: try to read from remote
remoteClient.readBatchData(stub, new FutureCallback<BulletinBoardMessage>() {
@Override
public void onSuccess(BulletinBoardMessage result) {
// Read from remote was successful: store in local and return result
localClient.postMessage(result, null);
if (callback != null)
callback.onSuccess(result);
}
@Override
public void onFailure(Throwable t) {
// Read from remote was unsuccessful: report error
if (callback != null)
callback.onFailure(t);
}
});
}
});
}
@Override
public void querySync(SyncQuery syncQuery, FutureCallback<SyncQueryResponse> callback) {
localClient.querySync(syncQuery, callback);
}
@Override
/**
* This is a stub method
* All resources are assumed to be initialized
*/
public void init(BulletinBoardClientParams clientParams) {}
@Override
public MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException {
return localClient.postMessage(msg);
}
@Override
public MessageID postAsBatch(BulletinBoardMessage msg, int chunkSize) throws CommunicationException {
MessageID result = localClient.postAsBatch(msg, chunkSize);
remoteClient.postAsBatch(msg, chunkSize);
return result;
}
@Override
public float getRedundancy(MessageID id) throws CommunicationException {
return remoteClient.getRedundancy(id);
}
@Override
public List<BulletinBoardMessage> readMessages(MessageFilterList filterList) throws CommunicationException {
subscriber.subscribe(filterList, new SubscriptionStoreCallback());
return localClient.readMessages(filterList);
}
@Override
public BulletinBoardMessage readMessage(MessageID msgID) throws CommunicationException {
BulletinBoardMessage result = null;
try {
result = localClient.readMessage(msgID);
} catch (CommunicationException e) {
//TODO: log
}
if (result == null){
result = remoteClient.readMessage(msgID);
if (result != null){
localClient.postMessage(result);
}
}
return result;
}
@Override
public BulletinBoardMessage readBatchData(BulletinBoardMessage stub) throws CommunicationException, IllegalArgumentException {
BulletinBoardMessage result = null;
try {
result = localClient.readBatchData(stub);
} catch (CommunicationException e) {
//TODO: log
}
if (result == null){
result = remoteClient.readBatchData(stub);
if (result != null){
localClient.postMessage(result);
}
}
return result;
}
@Override
public SyncQuery generateSyncQuery(GenerateSyncQueryParams generateSyncQueryParams) throws CommunicationException {
return localClient.generateSyncQuery(generateSyncQueryParams);
}
@Override
public void close() {
localClient.close();
remoteClient.close();
synchronizer.stop();
try {
syncThread.join();
} catch (InterruptedException e) {
//TODO: log interruption
}
}
@Override
public void subscribe(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
subscriber.subscribe(filterList, new SubscriptionStoreCallback(callback));
}
@Override
public void subscribe(MessageFilterList filterList, long startEntry, FutureCallback<List<BulletinBoardMessage>> callback) {
subscriber.subscribe(filterList, startEntry, new SubscriptionStoreCallback(callback));
}
}

29
bulletin-board-client/src/main/java/meerkat/bulletinboard/CachedClientBatchIdentifier.java Normal file
View File

@ -0,0 +1,29 @@
package meerkat.bulletinboard;
import meerkat.bulletinboard.AsyncBulletinBoardClient.BatchIdentifier;
import java.util.Arrays;
/**
* Created by Arbel Deutsch Peled on 17-Jun-16.
*/
public final class CachedClientBatchIdentifier implements BatchIdentifier {
// Per-server identifiers
private final BatchIdentifier localIdentifier;
private final BatchIdentifier remoteIdentifier;
public CachedClientBatchIdentifier(BatchIdentifier localIdentifier, BatchIdentifier remoteIdentifier) {
this.localIdentifier = localIdentifier;
this.remoteIdentifier = remoteIdentifier;
}
public BatchIdentifier getLocalIdentifier() {
return localIdentifier;
}
public BatchIdentifier getRemoteIdentifier() {
return remoteIdentifier;
}
}

689
bulletin-board-client/src/main/java/meerkat/bulletinboard/LocalBulletinBoardClient.java Normal file
View File

@ -0,0 +1,689 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.*;
import com.google.protobuf.Int64Value;
import com.google.protobuf.Timestamp;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageInputStream;
import meerkat.comm.MessageInputStream.MessageInputStreamFactory;
import meerkat.comm.MessageOutputStream;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto.Signature;
import meerkat.protobuf.Voting.*;
import meerkat.util.BulletinBoardUtils;
import javax.ws.rs.NotFoundException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* Created by Arbel Deutsch Peled on 15-Mar-16.
* This client wraps a BulletinBoardServer in an asynchronous client.
* It is meant to be used as a local cache handler and for testing purposes.
* This means the access to the server is direct (via method calls) instead of through a TCP connection.
* The client implements both synchronous and asynchronous method calls, but calls to the server itself are performed synchronously.
*/
public class LocalBulletinBoardClient implements DeletableSubscriptionBulletinBoardClient {
private final DeletableBulletinBoardServer server;
private final ListeningScheduledExecutorService executorService;
private final BulletinBoardDigest digest;
private final long subsrciptionDelay;
/**
* Initializes an instance of the client
* @param server an initialized Bulletin Board Server instance which will perform the actual processing of the requests
* @param threadNum is the number of concurrent threads to allocate for the client
*/
public LocalBulletinBoardClient(DeletableBulletinBoardServer server, int threadNum, int subscriptionDelay) {
this.server = server;
this.executorService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(threadNum));
this.digest = new GenericBulletinBoardDigest(new SHA256Digest());
this.subsrciptionDelay = subscriptionDelay;
}
private class MessagePoster implements Callable<Boolean> {
private final BulletinBoardMessage msg;
public MessagePoster(BulletinBoardMessage msg) {
this.msg = msg;
}
@Override
public Boolean call() throws CommunicationException {
return server.postMessage(msg).getValue();
}
}
@Override
public MessageID postMessage(BulletinBoardMessage msg, FutureCallback<Boolean> callback) {
Futures.addCallback(executorService.submit(new MessagePoster(msg)), callback);
digest.update(msg.getMsg());
return digest.digestAsMessageID();
}
private class CompleteBatchPoster implements Callable<Boolean> {
private final BulletinBoardMessage msg;
private final int chunkSize;
public CompleteBatchPoster(BulletinBoardMessage msg, int chunkSize) {
this.msg = msg;
this.chunkSize = chunkSize;
}
@Override
public Boolean call() throws CommunicationException {
BeginBatchMessage beginBatchMessage = BeginBatchMessage.newBuilder()
.addAllTag(msg.getMsg().getTagList())
.build();
Int64Value batchId = server.beginBatch(beginBatchMessage);
BatchMessage.Builder builder = BatchMessage.newBuilder()
.setBatchId(batchId.getValue());
List<BatchChunk> batchChunkList = BulletinBoardUtils.breakToBatch(msg, chunkSize);
int i=0;
for (BatchChunk chunk : batchChunkList){
server.postBatchMessage(builder.setSerialNum(i).setData(chunk).build());
i++;
}
CloseBatchMessage closeBatchMessage = BulletinBoardUtils.generateCloseBatchMessage(batchId, batchChunkList.size(), msg);
return server.closeBatch(closeBatchMessage).getValue();
}
}
@Override
public MessageID postAsBatch(BulletinBoardMessage msg, int chunkSize, FutureCallback<Boolean> callback) {
Futures.addCallback(executorService.submit(new CompleteBatchPoster(msg, chunkSize)), callback);
digest.reset();
digest.update(msg);
return digest.digestAsMessageID();
}
private class BatchBeginner implements Callable<SingleServerBatchIdentifier> {
private final BeginBatchMessage msg;
public BatchBeginner(BeginBatchMessage msg) {
this.msg = msg;
}
@Override
public SingleServerBatchIdentifier call() throws Exception {
return new SingleServerBatchIdentifier(server.beginBatch(msg));
}
}
@Override
public void beginBatch(Iterable<String> tags, FutureCallback<BatchIdentifier> callback) {
BeginBatchMessage beginBatchMessage = BeginBatchMessage.newBuilder()
.addAllTag(tags)
.build();
Futures.addCallback(executorService.submit(new BatchBeginner(beginBatchMessage)), callback);
}
private class BatchDataPoster implements Callable<Boolean> {
private final SingleServerBatchIdentifier batchId;
private final List<BatchChunk> batchChunkList;
private final int startPosition;
public BatchDataPoster(SingleServerBatchIdentifier batchId, List<BatchChunk> batchChunkList, int startPosition) {
this.batchId = batchId;
this.batchChunkList = batchChunkList;
this.startPosition = startPosition;
}
@Override
public Boolean call() throws Exception {
BatchMessage.Builder msgBuilder = BatchMessage.newBuilder()
.setBatchId(batchId.getBatchId().getValue());
int i = startPosition;
for (BatchChunk data : batchChunkList){
msgBuilder.setSerialNum(i)
.setData(data);
if (!server.postBatchMessage(msgBuilder.build()).getValue())
return false;
i++;
}
batchId.setLength(i);
return true;
}
}
@Override
public void postBatchData(BatchIdentifier batchId, List<BatchChunk> batchChunkList, int startPosition, FutureCallback<Boolean> callback)
throws IllegalArgumentException{
// Cast identifier to usable form
if (!(batchId instanceof SingleServerBatchIdentifier)){
throw new IllegalArgumentException("Error: batch identifier supplied was not created by this class.");
}
SingleServerBatchIdentifier identifier = (SingleServerBatchIdentifier) batchId;
// Add worker
Futures.addCallback(executorService.submit(new BatchDataPoster(identifier, batchChunkList, startPosition)), callback);
}
@Override
public void postBatchData(BatchIdentifier batchId, List<BatchChunk> batchChunkList, FutureCallback<Boolean> callback) throws IllegalArgumentException{
postBatchData(batchId, batchChunkList, 0, callback);
}
private class BatchCloser implements Callable<Boolean> {
private final CloseBatchMessage msg;
public BatchCloser(CloseBatchMessage msg) {
this.msg = msg;
}
@Override
public Boolean call() throws Exception {
return server.closeBatch(msg).getValue();
}
}
@Override
public void closeBatch(BatchIdentifier batchId, Timestamp timestamp, Iterable<Signature> signatures, FutureCallback<Boolean> callback) {
// Cast identifier to usable form
if (!(batchId instanceof SingleServerBatchIdentifier)){
throw new IllegalArgumentException("Error: batch identifier supplied was not created by this class.");
}
SingleServerBatchIdentifier identifier = (SingleServerBatchIdentifier) batchId;
// Add worker
CloseBatchMessage closeBatchMessage = CloseBatchMessage.newBuilder()
.setBatchId(identifier.getBatchId().getValue())
.setBatchLength(identifier.getLength())
.setTimestamp(timestamp)
.addAllSig(signatures)
.build();
Futures.addCallback(executorService.submit(new BatchCloser(closeBatchMessage)), callback);
}
private class RedundancyGetter implements Callable<Float> {
private final MessageID msgId;
public RedundancyGetter(MessageID msgId) {
this.msgId = msgId;
}
@Override
public Float call() throws Exception {
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(msgId.getID())
.build())
.build();
ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
MessageOutputStream<BulletinBoardMessage> outputStream = new MessageOutputStream<>(byteOutputStream);
server.readMessages(filterList,outputStream);
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(
new ByteArrayInputStream(byteOutputStream.toByteArray()),
BulletinBoardMessage.class);
if (inputStream.isAvailable())
return 1.0f;
else
return 0.0f;
}
}
@Override
public void getRedundancy(MessageID id, FutureCallback<Float> callback) {
Futures.addCallback(executorService.submit(new RedundancyGetter(id)), callback);
}
private class MessageReader implements Callable<List<BulletinBoardMessage>> {
private final MessageFilterList filterList;
public MessageReader(MessageFilterList filterList) {
this.filterList = filterList;
}
@Override
public List<BulletinBoardMessage> call() throws Exception {
ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
MessageOutputStream<BulletinBoardMessage> outputStream = new MessageOutputStream<>(byteOutputStream);
server.readMessages(filterList, outputStream);
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(
new ByteArrayInputStream(byteOutputStream.toByteArray()),
BulletinBoardMessage.class);
return inputStream.asList();
}
}
@Override
public void readMessages(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
Futures.addCallback(executorService.submit(new MessageReader(filterList)), callback);
}
class SubscriptionCallback implements FutureCallback<List<BulletinBoardMessage>> {
private MessageFilterList filterList;
private final FutureCallback<List<BulletinBoardMessage>> callback;
public SubscriptionCallback(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
this.filterList = filterList;
this.callback = callback;
}
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
// Report new messages to user
if (callback != null)
callback.onSuccess(result);
MessageFilterList.Builder filterBuilder = filterList.toBuilder();
// If any new messages arrived: update the MIN_ENTRY condition
if (result.size() > 0) {
// Remove last filter from list (MIN_ENTRY one)
filterBuilder.removeFilter(filterBuilder.getFilterCount() - 1);
// Add updated MIN_ENTRY filter (entry number is successor of last received entry's number)
filterBuilder.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MIN_ENTRY)
.setEntry(result.get(result.size() - 1).getEntryNum() + 1)
.build());
}
filterList = filterBuilder.build();
// Reschedule job
Futures.addCallback(executorService.schedule(new MessageReader(filterList), subsrciptionDelay, TimeUnit.MILLISECONDS), this);
}
@Override
public void onFailure(Throwable t) {
// Notify caller about failure and terminate subscription
if (callback != null)
callback.onFailure(t);
}
}
@Override
public void subscribe(MessageFilterList filterList, long startEntry, FutureCallback<List<BulletinBoardMessage>> callback) {
MessageFilterList subscriptionFilterList =
filterList.toBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MIN_ENTRY)
.setEntry(startEntry)
.build())
.build();
Futures.addCallback(executorService.submit(new MessageReader(subscriptionFilterList)), new SubscriptionCallback(subscriptionFilterList, callback));
}
@Override
public void subscribe(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
subscribe(filterList, 0, callback);
}
private class BatchDataReader implements Callable<List<BatchChunk>> {
private final MessageID msgID;
public BatchDataReader(MessageID msgID) {
this.msgID = msgID;
}
@Override
public List<BatchChunk> call() throws Exception {
BatchQuery batchQuery = BatchQuery.newBuilder()
.setMsgID(msgID)
.setStartPosition(0)
.build();
ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
MessageOutputStream<BatchChunk> batchOutputStream = new MessageOutputStream<>(byteOutputStream);
server.readBatch(batchQuery,batchOutputStream);
MessageInputStream<BatchChunk> inputStream =
MessageInputStreamFactory.createMessageInputStream(
new ByteArrayInputStream(byteOutputStream.toByteArray()),
BatchChunk.class);
return inputStream.asList();
}
}
private class CompleteBatchReader implements Callable<BulletinBoardMessage> {
private final MessageID msgID;
public CompleteBatchReader(MessageID msgID) {
this.msgID = msgID;
}
@Override
public BulletinBoardMessage call() throws Exception {
// Read message (mat be a stub)
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(msgID.getID())
.build())
.build();
MessageReader messageReader = new MessageReader(filterList);
List<BulletinBoardMessage> bulletinBoardMessages = messageReader.call();
if (bulletinBoardMessages.size() <= 0) {
throw new NotFoundException("Message does not exist");
}
BulletinBoardMessage msg = bulletinBoardMessages.get(0);
if (msg.getMsg().getDataTypeCase() == UnsignedBulletinBoardMessage.DataTypeCase.MSGID) {
// Read data
BatchDataReader batchDataReader = new BatchDataReader(msgID);
List<BatchChunk> batchChunkList = batchDataReader.call();
// Combine and return
return BulletinBoardUtils.gatherBatch(msg, batchChunkList);
} else {
return msg;
}
}
}
private class BatchDataCombiner implements Callable<BulletinBoardMessage> {
private final BulletinBoardMessage stub;
public BatchDataCombiner(BulletinBoardMessage stub) {
this.stub = stub;
}
@Override
public BulletinBoardMessage call() throws Exception {
MessageID msgID = MessageID.newBuilder().setID(stub.getMsg().getMsgId()).build();
BatchDataReader batchDataReader = new BatchDataReader(msgID);
List<BatchChunk> batchChunkList = batchDataReader.call();
return BulletinBoardUtils.gatherBatch(stub, batchChunkList);
}
}
@Override
public void readMessage(MessageID msgID, FutureCallback<BulletinBoardMessage> callback) {
Futures.addCallback(executorService.submit(new CompleteBatchReader(msgID)), callback);
}
@Override
public void readBatchData(BulletinBoardMessage stub, FutureCallback<BulletinBoardMessage> callback) throws IllegalArgumentException {
if (stub.getMsg().getDataTypeCase() != UnsignedBulletinBoardMessage.DataTypeCase.MSGID){
throw new IllegalArgumentException("Message is not a stub and does not contain the required message ID");
}
Futures.addCallback(executorService.submit(new BatchDataCombiner(stub)),callback);
}
private class SyncQueryHandler implements Callable<SyncQueryResponse> {
private final SyncQuery syncQuery;
public SyncQueryHandler(SyncQuery syncQuery) {
this.syncQuery = syncQuery;
}
@Override
public SyncQueryResponse call() throws Exception {
return server.querySync(syncQuery);
}
}
@Override
public void querySync(SyncQuery syncQuery, FutureCallback<SyncQueryResponse> callback) {
Futures.addCallback(executorService.submit(new SyncQueryHandler(syncQuery)), callback);
}
/**
* This method is a stub, since the implementation only considers one server, and that is given in the constructor
* @param ignored is ignored
*/
@Override
public void init(BulletinBoardClientParams ignored) {}
@Override
public MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException {
MessagePoster poster = new MessagePoster(msg);
poster.call();
digest.update(msg.getMsg());
return digest.digestAsMessageID();
}
@Override
public MessageID postAsBatch(BulletinBoardMessage msg, int chunkSize) throws CommunicationException {
CompleteBatchPoster poster = new CompleteBatchPoster(msg, chunkSize);
Boolean result = poster.call();
if (!result)
throw new CommunicationException("Batch post failed");
digest.reset();
digest.update(msg);
return digest.digestAsMessageID();
}
@Override
public float getRedundancy(MessageID id) {
try {
RedundancyGetter getter = new RedundancyGetter(id);
return getter.call();
} catch (Exception e) {
return -1.0f;
}
}
@Override
public List<BulletinBoardMessage> readMessages(MessageFilterList filterList) throws CommunicationException{
try {
MessageReader reader = new MessageReader(filterList);
return reader.call();
} catch (Exception e){
throw new CommunicationException("Error reading from server");
}
}
@Override
public BulletinBoardMessage readMessage(MessageID msgID) throws CommunicationException {
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(msgID.getID())
.build())
.build();
CompleteBatchReader completeBatchReader = new CompleteBatchReader(msgID);
try {
return completeBatchReader.call();
} catch (Exception e) {
throw new CommunicationException(e.getMessage() + " " + e.getMessage());
}
}
@Override
public BulletinBoardMessage readBatchData(BulletinBoardMessage stub) throws CommunicationException, IllegalArgumentException {
if (stub.getMsg().getDataTypeCase() != UnsignedBulletinBoardMessage.DataTypeCase.MSGID){
throw new IllegalArgumentException("Message is not a stub and does not contain the required message ID");
}
BatchDataCombiner combiner = new BatchDataCombiner(stub);
try {
return combiner.call();
} catch (Exception e) {
throw new CommunicationException(e.getCause() + " " + e.getMessage());
}
}
@Override
public SyncQuery generateSyncQuery(GenerateSyncQueryParams generateSyncQueryParams) throws CommunicationException {
return server.generateSyncQuery(generateSyncQueryParams);
}
@Override
public void deleteMessage(MessageID msgID, FutureCallback<Boolean> callback) {
try {
Boolean deleted = server.deleteMessage(msgID).getValue();
if (callback != null)
callback.onSuccess(deleted);
} catch (CommunicationException e) {
if (callback != null)
callback.onFailure(e);
}
}
@Override
public void deleteMessage(long entryNum, FutureCallback<Boolean> callback) {
try {
Boolean deleted = server.deleteMessage(entryNum).getValue();
if (callback != null)
callback.onSuccess(deleted);
} catch (CommunicationException e) {
if (callback != null)
callback.onFailure(e);
}
}
@Override
public boolean deleteMessage(MessageID msgID) throws CommunicationException {
return server.deleteMessage(msgID).getValue();
}
@Override
public boolean deleteMessage(long entryNum) throws CommunicationException {
return server.deleteMessage(entryNum).getValue();
}
@Override
public void close() {
try {
server.close();
} catch (CommunicationException ignored) {}
}
}

27
bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerBatchIdentifier.java Normal file
View File

@ -0,0 +1,27 @@
package meerkat.bulletinboard;
import meerkat.bulletinboard.AsyncBulletinBoardClient.BatchIdentifier;
import java.util.Arrays;
/**
* Created by Arbel Deutsch Peled on 17-Jun-16.
*/
public final class MultiServerBatchIdentifier implements AsyncBulletinBoardClient.BatchIdentifier {
// Per-server identifiers
private final Iterable<BatchIdentifier> identifiers;
public MultiServerBatchIdentifier(Iterable<BatchIdentifier> identifiers) {
this.identifiers = identifiers;
}
public MultiServerBatchIdentifier(BatchIdentifier[] identifiers) {
this.identifiers = Arrays.asList(identifiers);
}
public Iterable<BatchIdentifier> getIdentifiers() {
return identifiers;
}
}

98
bulletin-board-client/src/main/java/meerkat/bulletinboard/MultiServerWorker.java Normal file
View File

@ -0,0 +1,98 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.FutureCallback;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by Arbel Deutsch Peled on 09-Dec-15.
*
* This is a general class for handling multi-server work
* It utilizes Single Server Clients to perform the actual per-server work
*/
public abstract class MultiServerWorker<IN, OUT> extends BulletinClientWorker<IN> implements Runnable, FutureCallback<OUT>{
protected final List<SingleServerBulletinBoardClient> clients;
protected AtomicInteger minServers; // The minimal number of servers the job must be successful on for the job to be completed
protected AtomicInteger maxFailedServers; // The maximal number of allowed server failures
private AtomicBoolean returnedResult;
private final FutureCallback<OUT> futureCallback;
/**
* Constructor
* @param clients contains a list of Single Server clients to handle requests
* @param shuffleClients is a boolean stating whether or not it is needed to shuffle the clients
* @param minServers is the minimal amount of servers needed in order to successfully complete the job
* @param payload is the payload for the job
* @param maxRetry is the maximal per-server retry count
* @param futureCallback contains the callback methods used to report the result back to the client
*/
public MultiServerWorker(List<SingleServerBulletinBoardClient> clients, boolean shuffleClients,
int minServers, IN payload, int maxRetry,
FutureCallback<OUT> futureCallback) {
super(payload,maxRetry);
this.clients = clients;
if (shuffleClients){
Collections.shuffle(clients);
}
this.minServers = new AtomicInteger(minServers);
maxFailedServers = new AtomicInteger(clients.size() - minServers);
this.futureCallback = futureCallback;
returnedResult = new AtomicBoolean(false);
}
/**
* Constructor overload without client shuffling
*/
public MultiServerWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, IN payload, int maxRetry,
FutureCallback<OUT> futureCallback) {
this(clients, false, minServers, payload, maxRetry, futureCallback);
}
/**
* Used to report a successful operation to the client
* Only reports once to the client
* @param result is the result
*/
protected void succeed(OUT result){
if (returnedResult.compareAndSet(false, true)) {
if (futureCallback != null)
futureCallback.onSuccess(result);
}
}
/**
* Used to report a failed operation to the client
* Only reports once to the client
* @param t contains the error/exception that occurred
*/
protected void fail(Throwable t){
if (returnedResult.compareAndSet(false, true)) {
if (futureCallback != null)
futureCallback.onFailure(t);
}
}
protected int getClientNumber() {
return clients.size();
}
}

250
bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardClient.java
View File

@ -1,13 +1,16 @@
package meerkat.bulletinboard;
import com.google.protobuf.BoolValue;
import com.google.protobuf.ByteString;
import com.google.protobuf.Int64Value;
import meerkat.bulletinboard.workers.singleserver.*;
import meerkat.comm.CommunicationException;
import meerkat.crypto.Digest;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Voting;
import meerkat.protobuf.Voting.BulletinBoardClientParams;
import meerkat.protobuf.Voting.*;
import meerkat.rest.*;
import meerkat.util.BulletinBoardUtils;
import java.util.List;
@ -17,31 +20,35 @@ import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import static meerkat.bulletinboard.BulletinBoardConstants.*;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
* Implements BulletinBoardClient interface in a simple, straightforward manner
*/
public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
public class SimpleBulletinBoardClient implements BulletinBoardClient{
private List<String> meerkatDBs;
protected List<String> meerkatDBs;
private Client client;
protected Client client;
private Digest digest;
protected BulletinBoardDigest digest;
/**
* Stores database locations and initializes the web Client
* @param clientParams contains the data needed to access the DBs
*/
// @Override
public void init(Voting.BulletinBoardClientParams clientParams) {
@Override
public void init(BulletinBoardClientParams clientParams) {
meerkatDBs = clientParams.getBulletinBoardAddressList();
this.meerkatDBs = clientParams.getBulletinBoardAddressList();
client = ClientBuilder.newClient();
client.register(ProtobufMessageBodyReader.class);
client.register(ProtobufMessageBodyWriter.class);
digest = new SHA256Digest();
// Wrap the Digest into a BatchDigest
digest = new GenericBulletinBoardDigest(new SHA256Digest());
}
@ -52,23 +59,20 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
* @return the message ID for later retrieval
* @throws CommunicationException
*/
// @Override
@Override
public MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException {
WebTarget webTarget;
Response response;
Response response = null;
// Post message to all databases
try {
for (String db : meerkatDBs) {
webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.POST_MESSAGE_PATH);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF));
// Only consider valid responses
if (response.getStatusInfo() == Response.Status.OK
|| response.getStatusInfo() == Response.Status.CREATED) {
response.readEntity(BoolMsg.class).getValue();
}
SingleServerPostMessageWorker worker = new SingleServerPostMessageWorker(db, msg, 0);
worker.call();
}
} catch (Exception e) { // Occurs only when server replies with valid status but invalid data
throw new CommunicationException("Error accessing database: " + e.getMessage());
@ -88,7 +92,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
* @param id is the requested message ID
* @return the number of DBs in which retrieval was successful
*/
// @Override
@Override
public float getRedundancy(MessageID id) {
WebTarget webTarget;
Response response;
@ -104,7 +108,7 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
for (String db : meerkatDBs) {
try {
webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH);
webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF));
@ -123,39 +127,203 @@ public class SimpleBulletinBoardClient{ //implements BulletinBoardClient {
* If at the operation is successful for some DB: return the results and stop iterating
* If no operation is successful: return null (NOT blank list)
* @param filterList return only messages that match the filters (null means no filtering).
* @return
* @return the list of Bulletin Board messages that are returned from a server
*/
// @Override
public List<BulletinBoardMessage> readMessages(MessageFilterList filterList) {
WebTarget webTarget;
Response response;
BulletinBoardMessageList messageList;
@Override
public List<BulletinBoardMessage> readMessages(MessageFilterList filterList) throws CommunicationException{
// Replace null filter list with blank one.
if (filterList == null){
filterList = MessageFilterList.newBuilder().build();
filterList = MessageFilterList.getDefaultInstance();
}
String exceptionString = "";
for (String db : meerkatDBs) {
try {
webTarget = client.target(db).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF));
SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(db, filterList, 0);
messageList = response.readEntity(BulletinBoardMessageList.class);
List<BulletinBoardMessage> result = worker.call();
if (messageList != null){
return messageList.getMessageList();
}
return result;
} catch (Exception e) {}
} catch (Exception e) {
//TODO: log
exceptionString += e.getMessage() + "\n";
}
}
return null;
throw new CommunicationException("Could not find message in any DB. Errors follow:\n" + exceptionString);
}
@Override
public MessageID postAsBatch(BulletinBoardMessage msg, int chunkSize) throws CommunicationException {
List<BatchChunk> chunkList = BulletinBoardUtils.breakToBatch(msg, chunkSize);
BeginBatchMessage beginBatchMessage = BulletinBoardUtils.generateBeginBatchMessage(msg);
boolean posted = false;
// Post message to all databases
for (String db : meerkatDBs) {
try {
int pos = 0;
SingleServerBeginBatchWorker beginBatchWorker = new SingleServerBeginBatchWorker(db, beginBatchMessage, 0);
Int64Value batchId = beginBatchWorker.call();
BatchMessage.Builder builder = BatchMessage.newBuilder().setBatchId(batchId.getValue());
for (BatchChunk batchChunk : chunkList) {
SingleServerPostBatchWorker postBatchWorker =
new SingleServerPostBatchWorker(
db,
builder.setData(batchChunk).setSerialNum(pos).build(),
0);
postBatchWorker.call();
pos++;
}
CloseBatchMessage closeBatchMessage = BulletinBoardUtils.generateCloseBatchMessage(batchId, chunkList.size(), msg);
SingleServerCloseBatchWorker closeBatchWorker = new SingleServerCloseBatchWorker(db, closeBatchMessage, 0);
closeBatchWorker.call();
posted = true;
} catch(Exception ignored) {}
}
if (!posted){
throw new CommunicationException("Could not post to any server");
}
digest.reset();
digest.update(msg);
return digest.digestAsMessageID();
}
@Override
public BulletinBoardMessage readMessage(MessageID msgID) throws CommunicationException {
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(msgID.getID())
.build())
.build();
BatchQuery batchQuery = BatchQuery.newBuilder()
.setMsgID(msgID)
.setStartPosition(0)
.build();
String exceptionString = "";
for (String db : meerkatDBs) {
try {
SingleServerReadMessagesWorker messagesWorker = new SingleServerReadMessagesWorker(db, filterList, 0);
List<BulletinBoardMessage> messages = messagesWorker.call();
if (messages == null || messages.size() < 1)
continue;
BulletinBoardMessage stub = messages.get(0);
SingleServerReadBatchWorker batchWorker = new SingleServerReadBatchWorker(db, batchQuery, 0);
List<BatchChunk> batchChunkList = batchWorker.call();
return BulletinBoardUtils.gatherBatch(stub, batchChunkList);
} catch (Exception e) {
//TODO: log
exceptionString += e.getMessage() + "\n";
}
}
throw new CommunicationException("Could not find message in any DB. Errors follow:\n" + exceptionString);
}
@Override
public BulletinBoardMessage readBatchData(BulletinBoardMessage stub) throws CommunicationException, IllegalArgumentException {
if (stub.getMsg().getDataTypeCase() != UnsignedBulletinBoardMessage.DataTypeCase.MSGID){
throw new IllegalArgumentException("Message is not a stub and does not contain the required message ID");
}
BatchQuery batchQuery = BatchQuery.newBuilder()
.setMsgID(MessageID.newBuilder()
.setID(stub.getMsg().getMsgId())
.build())
.setStartPosition(0)
.build();
String exceptionString = "";
for (String db : meerkatDBs) {
try {
SingleServerReadBatchWorker batchWorker = new SingleServerReadBatchWorker(db, batchQuery, 0);
List<BatchChunk> batchChunkList = batchWorker.call();
return BulletinBoardUtils.gatherBatch(stub, batchChunkList);
} catch (Exception e) {
//TODO: log
exceptionString += e.getMessage() + "\n";
}
}
throw new CommunicationException("Could not find message in any DB. Errors follow:\n" + exceptionString);
}
@Override
public SyncQuery generateSyncQuery(GenerateSyncQueryParams generateSyncQueryParams) throws CommunicationException {
WebTarget webTarget;
Response response;
for (String db : meerkatDBs) {
try {
webTarget = client.target(db).path(BULLETIN_BOARD_SERVER_PATH).path(GENERATE_SYNC_QUERY_PATH);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(generateSyncQueryParams, Constants.MEDIATYPE_PROTOBUF));
return response.readEntity(SyncQuery.class);
} catch (Exception e) {}
}
throw new CommunicationException("Could not contact any server");
}
public void close() {
client.close();
}
// @Override
// public void registerNewMessageCallback(MessageCallback callback, MessageFilterList filterList) {
// callback.handleNewMessage(readMessages(filterList));
// }
}

241
bulletin-board-client/src/main/java/meerkat/bulletinboard/SimpleBulletinBoardSynchronizer.java Normal file
View File

@ -0,0 +1,241 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.ByteString;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.util.BulletinBoardUtils;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* Created by Arbel on 13/04/2016.
* Simple, straightforward implementation of the {@link BulletinBoardSynchronizer} interface
*/
public class SimpleBulletinBoardSynchronizer implements BulletinBoardSynchronizer {
private DeletableSubscriptionBulletinBoardClient localClient;
private AsyncBulletinBoardClient remoteClient;
private AtomicBoolean running;
private volatile SyncStatus syncStatus;
private List<FutureCallback<Integer>> messageCountCallbacks;
private List<FutureCallback<SyncStatus>> syncStatusCallbacks;
private static final MessageFilterList EMPTY_FILTER = MessageFilterList.getDefaultInstance();
private static final int DEFAULT_SLEEP_INTERVAL = 10000; // 10 Seconds
private static final int DEFAULT_WAIT_CAP = 300000; // 5 minutes wait before deciding that the sync has failed fatally
private final int SLEEP_INTERVAL;
private final int WAIT_CAP;
private Semaphore semaphore;
private class SyncCallback implements FutureCallback<List<BulletinBoardMessage>> {
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
// Notify Message Count callbacks if needed
if (syncStatus != SyncStatus.SYNCHRONIZED || result.size() > 0) {
for (FutureCallback<Integer> callback : messageCountCallbacks){
callback.onSuccess(result.size());
}
}
// Handle upload and status change
SyncStatus newStatus = SyncStatus.PENDING;
if (result.size() == 0) {
newStatus = SyncStatus.SYNCHRONIZED;
semaphore.release();
}
else{ // Upload messages
for (BulletinBoardMessage message : result){
try {
if (message.getMsg().getDataTypeCase() == UnsignedBulletinBoardMessage.DataTypeCase.MSGID) {
// This is a batch message: need to upload batch data as well as the message itself
BulletinBoardMessage completeMsg = localClient.readBatchData(message);
remoteClient.postMessage(completeMsg);
localClient.deleteMessage(completeMsg.getEntryNum());
} else {
// This is a regular message: post it
remoteClient.postMessage(message);
localClient.deleteMessage(message.getEntryNum());
}
} catch (CommunicationException e) {
// This is an error with the local server
// TODO: log
updateSyncStatus(SyncStatus.SERVER_ERROR);
}
}
}
updateSyncStatus(newStatus);
}
@Override
public void onFailure(Throwable t) {
updateSyncStatus(SyncStatus.SERVER_ERROR);
}
}
public SimpleBulletinBoardSynchronizer(int sleepInterval, int waitCap) {
this.syncStatus = SyncStatus.STOPPED;
this.SLEEP_INTERVAL = sleepInterval;
this.WAIT_CAP = waitCap;
this.running = new AtomicBoolean(false);
}
public SimpleBulletinBoardSynchronizer() {
this(DEFAULT_SLEEP_INTERVAL, DEFAULT_WAIT_CAP);
}
private synchronized void updateSyncStatus(SyncStatus newStatus) {
if (!running.get()) {
newStatus = SyncStatus.STOPPED;
}
if (newStatus != syncStatus){
syncStatus = newStatus;
for (FutureCallback<SyncStatus> callback : syncStatusCallbacks){
if (callback != null)
callback.onSuccess(syncStatus);
}
}
}
@Override
public void init(DeletableSubscriptionBulletinBoardClient localClient, AsyncBulletinBoardClient remoteClient) {
updateSyncStatus(SyncStatus.STOPPED);
this.localClient = localClient;
this.remoteClient = remoteClient;
messageCountCallbacks = new LinkedList<>();
syncStatusCallbacks = new LinkedList<>();
semaphore = new Semaphore(0);
}
@Override
public SyncStatus getSyncStatus() {
return syncStatus;
}
@Override
public void subscribeToSyncStatus(FutureCallback<SyncStatus> callback) {
syncStatusCallbacks.add(callback);
}
@Override
public List<BulletinBoardMessage> getRemainingMessages() throws CommunicationException{
return localClient.readMessages(EMPTY_FILTER);
}
@Override
public void getRemainingMessages(FutureCallback<List<BulletinBoardMessage>> callback) {
localClient.readMessages(EMPTY_FILTER, callback);
}
@Override
public long getRemainingMessagesCount() throws CommunicationException {
return localClient.readMessages(EMPTY_FILTER).size();
}
@Override
public void subscribeToRemainingMessagesCount(FutureCallback<Integer> callback) {
messageCountCallbacks.add(callback);
}
@Override
public void run() {
if (running.compareAndSet(false,true)){
updateSyncStatus(SyncStatus.PENDING);
SyncCallback callback = new SyncCallback();
while (syncStatus != SyncStatus.STOPPED) {
do {
localClient.readMessages(EMPTY_FILTER, callback);
try {
semaphore.tryAcquire(WAIT_CAP, TimeUnit.MILLISECONDS);
//TODO: log hard error. Too much time trying to upload data.
} catch (InterruptedException ignored) {
// We expect an interruption when the upload will complete
}
} while (syncStatus == SyncStatus.PENDING);
// Database is synced. Wait for new data.
try {
semaphore.tryAcquire(SLEEP_INTERVAL, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
//TODO: log (probably nudged)
}
}
}
}
@Override
public void nudge() {
semaphore.release();
}
@Override
public void stop() {
running.set(false);
updateSyncStatus(SyncStatus.STOPPED);
}
}

42
bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBatchIdentifier.java Normal file
View File

@ -0,0 +1,42 @@
package meerkat.bulletinboard;
import com.google.protobuf.Int64Value;
/**
* Created by Arbel Deutsch Peled on 16-Jun-16.
* Single-server implementation of the BatchIdentifier interface
*/
final class SingleServerBatchIdentifier implements AsyncBulletinBoardClient.BatchIdentifier {
private final Int64Value batchId;
private int length;
public SingleServerBatchIdentifier(Int64Value batchId) {
this.batchId = batchId;
length = 0;
}
public SingleServerBatchIdentifier(long batchId) {
this(Int64Value.newBuilder().setValue(batchId).build());
}
public Int64Value getBatchId() {
return batchId;
}
/**
* Overrides the existing length with the new one only if the new length is longer
* @param newLength
*/
public void setLength(int newLength) {
if (newLength > length) {
length = newLength;
}
}
public int getLength() {
return length;
}
}

875
bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerBulletinBoardClient.java Normal file
View File

@ -0,0 +1,875 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.Int64Value;
import com.google.protobuf.Timestamp;
import meerkat.bulletinboard.workers.singleserver.*;
import meerkat.comm.CommunicationException;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto;
import meerkat.protobuf.Voting.BulletinBoardClientParams;
import meerkat.util.BulletinBoardUtils;
import javax.ws.rs.client.Client;
import java.lang.Iterable;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by Arbel Deutsch Peled on 28-Dec-15.
*
* This class implements the asynchronous Bulletin Board Client interface
* It only handles a single Bulletin Board Server
* If the list of servers contains more than one server: the server actually used is the first one
* The class further implements a delayed access to the server after a communication error occurs
*/
public class SingleServerBulletinBoardClient implements SubscriptionBulletinBoardClient {
protected Client client;
protected BulletinBoardDigest digest;
private String dbAddress;
private final int MAX_RETRIES = 11;
private final ListeningScheduledExecutorService executorService;
private long lastServerErrorTime;
private final long FAIL_DELAY_IN_MILLISECONDS;
private final long SUBSCRIPTION_INTERVAL_IN_MILLISECONDS;
/**
* Notify the client that a job has failed
* This makes new scheduled jobs be scheduled for a later time (after the given delay)
*/
protected void fail() {
// Update last fail time
lastServerErrorTime = System.currentTimeMillis();
}
private class SynchronousRetry<OUT> {
private final SingleServerWorker<?,OUT> worker;
private String thrown;
public SynchronousRetry(SingleServerWorker<?,OUT> worker) {
this.worker = worker;
this.thrown = "Could not contact server. Errors follow:\n";
}
OUT run() throws CommunicationException {
do {
try {
return worker.call();
} catch (Exception e) {
thrown += e.getCause() + " " + e.getMessage() + "\n";
}
try {
Thread.sleep(FAIL_DELAY_IN_MILLISECONDS);
} catch (InterruptedException e) {
//TODO: log
}
worker.decMaxRetry();
} while (worker.isRetry());
throw new CommunicationException(thrown);
}
}
/**
* This method adds a worker to the scheduled queue of the threadpool
* If the server is in an accessible state: the job is submitted for immediate handling
* If the server is not accessible: the job is scheduled for a later time
* @param worker is the worker that should be scheduled for work
* @param callback is the class containing callbacks for handling job completion/failure
*/
protected void scheduleWorker(SingleServerWorker worker, FutureCallback callback){
long timeSinceLastServerError = System.currentTimeMillis() - lastServerErrorTime;
if (timeSinceLastServerError >= FAIL_DELAY_IN_MILLISECONDS) {
// Schedule for immediate processing
Futures.addCallback(executorService.submit(worker), callback);
} else {
// Schedule for processing immediately following delay expiry
Futures.addCallback(executorService.schedule(
worker,
FAIL_DELAY_IN_MILLISECONDS - timeSinceLastServerError,
TimeUnit.MILLISECONDS),
callback);
}
}
/**
* Inner class for handling simple operation results and retrying if needed
*/
class RetryCallback<T> implements FutureCallback<T> {
private final SingleServerWorker worker;
private final FutureCallback<T> futureCallback;
public RetryCallback(SingleServerWorker worker, FutureCallback<T> futureCallback) {
this.worker = worker;
this.futureCallback = futureCallback;
}
@Override
public void onSuccess(T result) {
if (futureCallback != null)
futureCallback.onSuccess(result);
}
@Override
public void onFailure(Throwable t) {
// Notify client about failure
fail();
// Check if another attempt should be made
worker.decMaxRetry();
if (worker.isRetry()) {
// Perform another attempt
scheduleWorker(worker, this);
} else {
// No more retries: notify caller about failure
if (futureCallback != null)
futureCallback.onFailure(t);
}
}
}
/**
* This callback ties together all the per-batch-data callbacks into a single callback
* It reports success back to the user only if all of the batch-data were successfully posted
* If any batch-data fails to post: this callback reports failure
*/
class PostBatchChunkListCallback implements FutureCallback<Boolean> {
private final FutureCallback<Boolean> callback;
private AtomicInteger batchDataRemaining;
private AtomicBoolean aggregatedResult;
public PostBatchChunkListCallback(int batchDataLength, FutureCallback<Boolean> callback) {
this.callback = callback;
this.batchDataRemaining = new AtomicInteger(batchDataLength);
this.aggregatedResult = new AtomicBoolean(false);
}
@Override
public void onSuccess(Boolean result) {
if (result){
this.aggregatedResult.set(true);
}
if (batchDataRemaining.decrementAndGet() == 0){
if (callback != null)
callback.onSuccess(this.aggregatedResult.get());
}
}
@Override
public void onFailure(Throwable t) {
// Notify caller about failure
if (callback != null)
callback.onFailure(t);
}
}
private class ReadBatchCallback implements FutureCallback<List<BatchChunk>> {
private final BulletinBoardMessage stub;
private final FutureCallback<BulletinBoardMessage> callback;
public ReadBatchCallback(BulletinBoardMessage stub, FutureCallback<BulletinBoardMessage> callback) {
this.stub = stub;
this.callback = callback;
}
@Override
public void onSuccess(List<BatchChunk> result) {
callback.onSuccess(BulletinBoardUtils.gatherBatch(stub, result));
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
}
/**
* This callback receives a message which may be a stub
* If the message is not a stub: it returns it as is to a callback function
* If it is a stub: it schedules a read of the batch data which will return a complete message to the callback function
*/
class CompleteMessageReadCallback implements FutureCallback<List<BulletinBoardMessage>>{
private final FutureCallback<BulletinBoardMessage> callback;
public CompleteMessageReadCallback(FutureCallback<BulletinBoardMessage> callback) {
this.callback = callback;
}
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
if (result.size() <= 0) {
onFailure(new CommunicationException("Could not find required message on the server."));
} else {
BulletinBoardMessage msg = result.get(0);
if (msg.getMsg().getDataTypeCase() != UnsignedBulletinBoardMessage.DataTypeCase.MSGID) {
callback.onSuccess(msg);
} else {
// Create job with MAX retries for retrieval of the Batch Data List
BatchQuery batchQuery = BatchQuery.newBuilder()
.setMsgID(MessageID.newBuilder()
.setID(msg.getMsg().getMsgId())
.build())
.build();
SingleServerReadBatchWorker batchWorker = new SingleServerReadBatchWorker(dbAddress, batchQuery, MAX_RETRIES);
scheduleWorker(batchWorker, new ReadBatchCallback(msg, callback));
}
}
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
}
/**
* Inner class for handling returned values of subscription operations
* This class's methods also ensure continued operation of the subscription
*/
class SubscriptionCallback implements FutureCallback<List<BulletinBoardMessage>> {
private SingleServerReadMessagesWorker worker;
private final FutureCallback<List<BulletinBoardMessage>> callback;
private MessageFilterList.Builder filterBuilder;
public SubscriptionCallback(SingleServerReadMessagesWorker worker, FutureCallback<List<BulletinBoardMessage>> callback) {
this.worker = worker;
this.callback = callback;
filterBuilder = worker.getPayload().toBuilder();
}
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
// Report new messages to user
if (callback != null)
callback.onSuccess(result);
// Update filter if needed
if (result.size() > 0) {
// Remove last filter from list (MIN_ENTRY one)
filterBuilder.removeFilter(filterBuilder.getFilterCount() - 1);
// Add updated MIN_ENTRY filter (entry number is successor of last received entry's number)
filterBuilder.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MIN_ENTRY)
.setEntry(result.get(result.size() - 1).getEntryNum() + 1)
.build());
}
// Create new worker with updated task
worker = new SingleServerReadMessagesWorker(worker.serverAddress, filterBuilder.build(), MAX_RETRIES);
RetryCallback<List<BulletinBoardMessage>> retryCallback = new RetryCallback<>(worker, this);
// Schedule the worker to run after the given interval has elapsed
Futures.addCallback(executorService.schedule(worker, SUBSCRIPTION_INTERVAL_IN_MILLISECONDS, TimeUnit.MILLISECONDS), retryCallback);
}
@Override
public void onFailure(Throwable t) {
// Notify client about failure
fail();
// Notify caller about failure and terminate subscription
if (callback != null)
callback.onFailure(t);
}
}
public SingleServerBulletinBoardClient(ListeningScheduledExecutorService executorService,
long failDelayInMilliseconds,
long subscriptionIntervalInMilliseconds) {
this.executorService = executorService;
this.FAIL_DELAY_IN_MILLISECONDS = failDelayInMilliseconds;
this.SUBSCRIPTION_INTERVAL_IN_MILLISECONDS = subscriptionIntervalInMilliseconds;
// Set server error time to a time sufficiently in the past to make new jobs go through
lastServerErrorTime = System.currentTimeMillis() - failDelayInMilliseconds;
}
public SingleServerBulletinBoardClient(int threadPoolSize, long failDelayInMilliseconds, long subscriptionIntervalInMilliseconds) {
this(MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(threadPoolSize)),
failDelayInMilliseconds,
subscriptionIntervalInMilliseconds);
}
/**
* Stores database location, initializes the web Client and
* @param clientParams contains the data needed to access the DBs
*/
@Override
public void init(BulletinBoardClientParams clientParams) {
this.digest = new GenericBulletinBoardDigest(new SHA256Digest());
// Remove all but first DB address
this.dbAddress = clientParams.getBulletinBoardAddress(0);
}
// Synchronous methods
@Override
public MessageID postMessage(BulletinBoardMessage msg) throws CommunicationException {
SingleServerPostMessageWorker worker = new SingleServerPostMessageWorker(dbAddress, msg, MAX_RETRIES);
SynchronousRetry<Boolean> retry = new SynchronousRetry<>(worker);
retry.run();
digest.reset();
digest.update(msg);
return digest.digestAsMessageID();
}
@Override
public float getRedundancy(MessageID id) throws CommunicationException {
SingleServerGetRedundancyWorker worker = new SingleServerGetRedundancyWorker(dbAddress, id, MAX_RETRIES);
SynchronousRetry<Float> retry = new SynchronousRetry<>(worker);
return retry.run();
}
@Override
public List<BulletinBoardMessage> readMessages(MessageFilterList filterList) throws CommunicationException {
SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(dbAddress, filterList, MAX_RETRIES);
SynchronousRetry<List<BulletinBoardMessage>> retry = new SynchronousRetry<>(worker);
return retry.run();
}
@Override
public MessageID postAsBatch(BulletinBoardMessage msg, int chunkSize) throws CommunicationException {
// Begin the batch and obtain identifier
BeginBatchMessage beginBatchMessage = BeginBatchMessage.newBuilder()
.addAllTag(msg.getMsg().getTagList())
.build();
SingleServerBeginBatchWorker beginBatchWorker = new SingleServerBeginBatchWorker(dbAddress, beginBatchMessage, MAX_RETRIES);
SynchronousRetry<Int64Value> beginRetry = new SynchronousRetry<>(beginBatchWorker);
Int64Value identifier = beginRetry.run();
// Post data chunks
List<BatchChunk> batchChunkList = BulletinBoardUtils.breakToBatch(msg, chunkSize);
BatchMessage.Builder builder = BatchMessage.newBuilder().setBatchId(identifier.getValue());
int position = 0;
for (BatchChunk data : batchChunkList) {
builder.setSerialNum(position).setData(data);
SingleServerPostBatchWorker dataWorker = new SingleServerPostBatchWorker(dbAddress, builder.build(), MAX_RETRIES);
SynchronousRetry<Boolean> dataRetry = new SynchronousRetry<>(dataWorker);
dataRetry.run();
// Increment position in batch
position++;
}
// Close batch
CloseBatchMessage closeBatchMessage = CloseBatchMessage.newBuilder()
.setBatchId(identifier.getValue())
.addAllSig(msg.getSigList())
.setTimestamp(msg.getMsg().getTimestamp())
.setBatchLength(position)
.build();
SingleServerCloseBatchWorker closeBatchWorker = new SingleServerCloseBatchWorker(dbAddress, closeBatchMessage, MAX_RETRIES);
SynchronousRetry<Boolean> retry = new SynchronousRetry<>(closeBatchWorker);
retry.run();
// Calculate ID and return
digest.reset();
digest.update(msg);
return digest.digestAsMessageID();
}
@Override
public BulletinBoardMessage readMessage(MessageID msgID) throws CommunicationException {
// Retrieve message (which may be a stub)
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(msgID.getID())
.build())
.build();
SingleServerReadMessagesWorker stubWorker = new SingleServerReadMessagesWorker(dbAddress, filterList, MAX_RETRIES);
SynchronousRetry<List<BulletinBoardMessage>> retry = new SynchronousRetry<>(stubWorker);
List<BulletinBoardMessage> messages = retry.run();
if (messages.size() <= 0) {
throw new CommunicationException("Could not find message in database.");
}
BulletinBoardMessage msg = messages.get(0);
if (msg.getMsg().getDataTypeCase() != UnsignedBulletinBoardMessage.DataTypeCase.MSGID) {
// We retrieved a complete message. Return it.
return msg;
} else {
// We retrieved a stub. Retrieve data.
return readBatchData(msg);
}
}
@Override
public BulletinBoardMessage readBatchData(BulletinBoardMessage stub) throws CommunicationException, IllegalArgumentException {
BatchQuery batchQuery = BatchQuery.newBuilder()
.setMsgID(MessageID.newBuilder()
.setID(stub.getMsg().getMsgId())
.build())
.setStartPosition(0)
.build();
SingleServerReadBatchWorker readBatchWorker = new SingleServerReadBatchWorker(dbAddress, batchQuery, MAX_RETRIES);
SynchronousRetry<List<BatchChunk>> batchRetry = new SynchronousRetry<>(readBatchWorker);
List<BatchChunk> batchChunkList = batchRetry.run();
return BulletinBoardUtils.gatherBatch(stub, batchChunkList);
}
@Override
public SyncQuery generateSyncQuery(GenerateSyncQueryParams generateSyncQueryParams) throws CommunicationException {
SingleServerGenerateSyncQueryWorker worker =
new SingleServerGenerateSyncQueryWorker(dbAddress, generateSyncQueryParams, MAX_RETRIES);
SynchronousRetry<SyncQuery> retry = new SynchronousRetry<>(worker);
return retry.run();
}
// Asynchronous methods
@Override
public MessageID postMessage(BulletinBoardMessage msg, FutureCallback<Boolean> callback) {
// Create worker with redundancy 1 and MAX_RETRIES retries
SingleServerPostMessageWorker worker = new SingleServerPostMessageWorker(dbAddress, msg, MAX_RETRIES);
// Submit worker and create callback
scheduleWorker(worker, new RetryCallback<>(worker, callback));
// Calculate the correct message ID and return it
digest.reset();
digest.update(msg.getMsg());
return digest.digestAsMessageID();
}
private class PostBatchDataCallback implements FutureCallback<Boolean> {
private final BulletinBoardMessage msg;
private final BatchIdentifier identifier;
private final FutureCallback<Boolean> callback;
public PostBatchDataCallback(BulletinBoardMessage msg, BatchIdentifier identifier, FutureCallback<Boolean> callback) {
this.msg = msg;
this.identifier = identifier;
this.callback = callback;
}
@Override
public void onSuccess(Boolean result) {
closeBatch(
identifier,
msg.getMsg().getTimestamp(),
msg.getSigList(),
callback
);
}
@Override
public void onFailure(Throwable t) {
if (callback != null)
callback.onFailure(t);
}
}
private class ContinueBatchCallback implements FutureCallback<BatchIdentifier> {
private final BulletinBoardMessage msg;
private final int chunkSize;
private final FutureCallback<Boolean> callback;
public ContinueBatchCallback(BulletinBoardMessage msg, int chunkSize, FutureCallback<Boolean> callback) {
this.msg = msg;
this.chunkSize = chunkSize;
this.callback = callback;
}
@Override
public void onSuccess(BatchIdentifier identifier) {
List<BatchChunk> batchChunkList = BulletinBoardUtils.breakToBatch(msg, chunkSize);
postBatchData(
identifier,
batchChunkList,
0,
new PostBatchDataCallback(msg, identifier, callback));
}
@Override
public void onFailure(Throwable t) {
if (callback != null)
callback.onFailure(t);
}
}
@Override
public MessageID postAsBatch(BulletinBoardMessage msg, int chunkSize, FutureCallback<Boolean> callback) {
beginBatch(
msg.getMsg().getTagList(),
new ContinueBatchCallback(msg, chunkSize, callback)
);
digest.update(msg);
return digest.digestAsMessageID();
}
private class BeginBatchCallback implements FutureCallback<Int64Value> {
private final FutureCallback<BatchIdentifier> callback;
public BeginBatchCallback(FutureCallback<BatchIdentifier> callback) {
this.callback = callback;
}
@Override
public void onSuccess(Int64Value result) {
callback.onSuccess(new SingleServerBatchIdentifier(result));
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
}
@Override
public void beginBatch(Iterable<String> tags, FutureCallback<BatchIdentifier> callback) {
BeginBatchMessage beginBatchMessage = BeginBatchMessage.newBuilder()
.addAllTag(tags)
.build();
// Create worker with redundancy 1 and MAX_RETRIES retries
SingleServerBeginBatchWorker worker =
new SingleServerBeginBatchWorker(dbAddress, beginBatchMessage, MAX_RETRIES);
// Submit worker and create callback
scheduleWorker(worker, new RetryCallback<>(worker, new BeginBatchCallback(callback)));
}
@Override
public void postBatchData(BatchIdentifier batchIdentifier, List<BatchChunk> batchChunkList,
int startPosition, FutureCallback<Boolean> callback) throws IllegalArgumentException{
// Cast identifier to usable form
if (!(batchIdentifier instanceof SingleServerBatchIdentifier)){
throw new IllegalArgumentException("Error: batch identifier supplied was not created by this class.");
}
SingleServerBatchIdentifier identifier = (SingleServerBatchIdentifier) batchIdentifier;
// Update batch size
identifier.setLength(startPosition + batchChunkList.size());
// Create a unified callback to aggregate successful posts
PostBatchChunkListCallback listCallback = new PostBatchChunkListCallback(batchChunkList.size(), callback);
// Iterate through data list
BatchMessage.Builder builder = BatchMessage.newBuilder()
.setBatchId(identifier.getBatchId().getValue());
for (BatchChunk data : batchChunkList) {
builder.setSerialNum(startPosition).setData(data);
// Create worker with redundancy 1 and MAX_RETRIES retries
SingleServerPostBatchWorker worker =
new SingleServerPostBatchWorker(dbAddress, builder.build(), MAX_RETRIES);
// Create worker with redundancy 1 and MAX_RETRIES retries
scheduleWorker(worker, new RetryCallback<>(worker, listCallback));
// Increment position in batch
startPosition++;
}
}
@Override
public void postBatchData(BatchIdentifier batchIdentifier, List<BatchChunk> batchChunkList, FutureCallback<Boolean> callback)
throws IllegalArgumentException {
postBatchData(batchIdentifier, batchChunkList, 0, callback);
}
@Override
public void closeBatch(BatchIdentifier batchIdentifier, Timestamp timestamp, Iterable<Crypto.Signature> signatures, FutureCallback<Boolean> callback)
throws IllegalArgumentException {
if (!(batchIdentifier instanceof SingleServerBatchIdentifier)){
throw new IllegalArgumentException("Error: batch identifier supplied was not created by this class.");
}
SingleServerBatchIdentifier identifier = (SingleServerBatchIdentifier) batchIdentifier;
CloseBatchMessage closeBatchMessage = CloseBatchMessage.newBuilder()
.setBatchId(identifier.getBatchId().getValue())
.setBatchLength(identifier.getLength())
.setTimestamp(timestamp)
.addAllSig(signatures)
.build();
// Create worker with redundancy 1 and MAX_RETRIES retries
SingleServerCloseBatchWorker worker =
new SingleServerCloseBatchWorker(dbAddress, closeBatchMessage, MAX_RETRIES);
// Submit worker and create callback
scheduleWorker(worker, new RetryCallback<>(worker, callback));
}
@Override
public void getRedundancy(MessageID id, FutureCallback<Float> callback) {
// Create worker with no retries
SingleServerGetRedundancyWorker worker = new SingleServerGetRedundancyWorker(dbAddress, id, 1);
// Submit job and create callback
scheduleWorker(worker, new RetryCallback<>(worker, callback));
}
@Override
public void readMessages(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
// Create job with no retries
SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(dbAddress, filterList, 1);
// Submit job and create callback
scheduleWorker(worker, new RetryCallback<>(worker, callback));
}
@Override
public void readMessage(MessageID msgID, FutureCallback<BulletinBoardMessage> callback) {
// Create job with MAX retries for retrieval of the Bulletin Board Message (which may be a stub)
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(msgID.getID())
.build())
.build();
BatchQuery batchQuery = BatchQuery.newBuilder()
.setMsgID(msgID)
.setStartPosition(0)
.build();
SingleServerReadMessagesWorker messageWorker = new SingleServerReadMessagesWorker(dbAddress, filterList, MAX_RETRIES);
// Submit jobs with wrapped callbacks
scheduleWorker(messageWorker, new RetryCallback<>(messageWorker, new CompleteMessageReadCallback(callback)));
}
@Override
public void readBatchData(BulletinBoardMessage stub, FutureCallback<BulletinBoardMessage> callback) throws IllegalArgumentException{
if (stub.getMsg().getDataTypeCase() != UnsignedBulletinBoardMessage.DataTypeCase.MSGID) {
throw new IllegalArgumentException("Message is not a stub and does not contain the required message ID");
}
// Create job with MAX retries for retrieval of the Batch Data List
BatchQuery batchQuery = BatchQuery.newBuilder()
.setMsgID(MessageID.newBuilder()
.setID(stub.getMsg().getMsgId())
.build())
.setStartPosition(0)
.build();
SingleServerReadBatchWorker batchWorker = new SingleServerReadBatchWorker(dbAddress, batchQuery, MAX_RETRIES);
scheduleWorker(batchWorker, new RetryCallback<>(batchWorker, new ReadBatchCallback(stub, callback)));
}
@Override
public void querySync(SyncQuery syncQuery, FutureCallback<SyncQueryResponse> callback) {
SingleServerQuerySyncWorker worker = new SingleServerQuerySyncWorker(dbAddress, syncQuery, MAX_RETRIES);
scheduleWorker(worker, new RetryCallback<>(worker, callback));
}
@Override
public void subscribe(MessageFilterList filterList, long startEntry, FutureCallback<List<BulletinBoardMessage>> callback) {
// Remove all existing MIN_ENTRY filters and create new one that starts at 0
MessageFilterList.Builder filterListBuilder = filterList.toBuilder();
Iterator<MessageFilter> iterator = filterListBuilder.getFilterList().iterator();
while (iterator.hasNext()) {
MessageFilter filter = iterator.next();
if (filter.getType() == FilterType.MIN_ENTRY){
iterator.remove();
}
}
filterListBuilder.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MIN_ENTRY)
.setEntry(startEntry)
.build());
// Create job with no retries
SingleServerReadMessagesWorker worker = new SingleServerReadMessagesWorker(dbAddress, filterListBuilder.build(), MAX_RETRIES);
// Submit job and create callback that retries on failure and handles repeated subscription
scheduleWorker(worker, new RetryCallback<>(worker, new SubscriptionCallback(worker, callback)));
}
@Override
public void subscribe(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
subscribe(filterList, 0, callback);
}
@Override
public void close() {
executorService.shutdown();
}
}

39
bulletin-board-client/src/main/java/meerkat/bulletinboard/SingleServerWorker.java Normal file
View File

@ -0,0 +1,39 @@
package meerkat.bulletinboard;
import meerkat.rest.ProtobufMessageBodyReader;
import meerkat.rest.ProtobufMessageBodyWriter;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import java.util.concurrent.Callable;
/**
* Created by Arbel Deutsch Peled on 02-Jan-16.
*/
public abstract class SingleServerWorker<IN, OUT> extends BulletinClientWorker<IN> implements Callable<OUT>{
// This resource enabled creation of a single Client per thread.
protected static final ThreadLocal<Client> clientLocal =
new ThreadLocal<Client> () {
@Override protected Client initialValue() {
Client client;
client = ClientBuilder.newClient();
client.register(ProtobufMessageBodyReader.class);
client.register(ProtobufMessageBodyWriter.class);
return client;
}
};
protected final String serverAddress;
public SingleServerWorker(String serverAddress, IN payload, int maxRetry) {
super(payload, maxRetry);
this.serverAddress = serverAddress;
}
public String getServerAddress() {
return serverAddress;
}
}

249
bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardClient.java
View File

@ -1,42 +1,62 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.*;
import com.google.protobuf.ByteString;
import meerkat.bulletinboard.callbacks.GetRedundancyFutureCallback;
import meerkat.bulletinboard.callbacks.PostMessageFutureCallback;
import meerkat.bulletinboard.callbacks.ReadMessagesFutureCallback;
import meerkat.comm.CommunicationException;
import meerkat.crypto.Digest;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Voting;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.Timestamp;
import meerkat.bulletinboard.workers.multiserver.*;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto.Signature;
import meerkat.protobuf.Voting.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
* Thread-based implementation of a Bulletin Board Client.
* Thread-based implementation of a Async Bulletin Board Client.
* Features:
* 1. Handles tasks concurrently.
* 2. Retries submitting
*/
public class ThreadedBulletinBoardClient implements BulletinBoardClient {
public class ThreadedBulletinBoardClient extends SimpleBulletinBoardClient implements AsyncBulletinBoardClient {
private final static int THREAD_NUM = 10;
ListeningExecutorService listeningExecutor;
// Executor service for handling jobs
private final static int JOBS_THREAD_NUM = 5;
private ExecutorService executorService;
private Digest digest;
// Per-server clients
private List<SingleServerBulletinBoardClient> clients;
private List<String> meerkatDBs;
private String postSubAddress;
private String readSubAddress;
private BulletinBoardDigest batchDigest;
private final static int POST_MESSAGE_RETRY_NUM = 3;
private final static int READ_MESSAGES_RETRY_NUM = 1;
private final static int GET_REDUNDANCY_RETRY_NUM = 1;
private final int SERVER_THREADPOOL_SIZE;
private final long FAIL_DELAY;
private final long SUBSCRIPTION_INTERVAL;
private static final int DEFAULT_SERVER_THREADPOOL_SIZE = 5;
private static final long DEFAULT_FAIL_DELAY = 5000;
private static final long DEFAULT_SUBSCRIPTION_INTERVAL = 10000;
private int minAbsoluteRedundancy;
public ThreadedBulletinBoardClient(int serverThreadpoolSize, long failDelay, long subscriptionInterval) {
SERVER_THREADPOOL_SIZE = serverThreadpoolSize;
FAIL_DELAY = failDelay;
SUBSCRIPTION_INTERVAL = subscriptionInterval;
}
public ThreadedBulletinBoardClient() {
this(DEFAULT_SERVER_THREADPOOL_SIZE, DEFAULT_FAIL_DELAY, DEFAULT_SUBSCRIPTION_INTERVAL);
}
/**
* Stores database locations and initializes the web Client
* Stores the required minimum redundancy.
@ -44,15 +64,29 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient {
* @param clientParams contains the required information
*/
@Override
public void init(Voting.BulletinBoardClientParams clientParams) {
public void init(BulletinBoardClientParams clientParams) {
meerkatDBs = clientParams.getBulletinBoardAddressList();
super.init(clientParams);
minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * meerkatDBs.size());
batchDigest = new GenericBulletinBoardDigest(digest);
listeningExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(THREAD_NUM));
minAbsoluteRedundancy = (int) (clientParams.getMinRedundancy() * (float) clientParams.getBulletinBoardAddressCount());
digest = new SHA256Digest();
executorService = Executors.newFixedThreadPool(JOBS_THREAD_NUM);
clients = new ArrayList<>(clientParams.getBulletinBoardAddressCount());
for (String address : clientParams.getBulletinBoardAddressList()){
SingleServerBulletinBoardClient client =
new SingleServerBulletinBoardClient(SERVER_THREADPOOL_SIZE, FAIL_DELAY, SUBSCRIPTION_INTERVAL);
client.init(BulletinBoardClientParams.newBuilder()
.addBulletinBoardAddress(address)
.build());
clients.add(client);
}
}
@ -61,21 +95,101 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient {
* Retry failed DBs
* @param msg is the message,
* @return the message ID for later retrieval
* @throws CommunicationException
*/
@Override
public MessageID postMessage(BulletinBoardMessage msg, ClientCallback<?> callback){
public MessageID postMessage(BulletinBoardMessage msg, FutureCallback<Boolean> callback){
// Create job
BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.POST_MESSAGE, msg, -1);
MultiServerPostMessageWorker worker =
new MultiServerPostMessageWorker(clients, minAbsoluteRedundancy, msg, POST_MESSAGE_RETRY_NUM, callback);
// Submit job and create callback
Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new PostMessageFutureCallback(listeningExecutor, callback));
// Submit job
executorService.submit(worker);
// Calculate the correct message ID and return it
digest.reset();
digest.update(msg.getMsg());
return MessageID.newBuilder().setID(ByteString.copyFrom(digest.digest())).build();
batchDigest.reset();
batchDigest.update(msg.getMsg());
return batchDigest.digestAsMessageID();
}
@Override
public MessageID postAsBatch(BulletinBoardMessage msg, int chunkSize, FutureCallback<Boolean> callback) {
// Create job
MultiServerPostBatchWorker worker =
new MultiServerPostBatchWorker(clients, minAbsoluteRedundancy, msg, chunkSize, POST_MESSAGE_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
// Calculate the correct message ID and return it
batchDigest.reset();
batchDigest.update(msg);
return batchDigest.digestAsMessageID();
}
@Override
public void beginBatch(Iterable<String> tags, FutureCallback<BatchIdentifier> callback) {
// Create job
MultiServerBeginBatchWorker worker =
new MultiServerBeginBatchWorker(clients, minAbsoluteRedundancy, tags, POST_MESSAGE_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
}
@Override
public void postBatchData(BatchIdentifier batchIdentifier, List<BatchChunk> batchChunkList,
int startPosition, FutureCallback<Boolean> callback) throws IllegalArgumentException {
// Cast identifier to usable form
if (!(batchIdentifier instanceof MultiServerBatchIdentifier)){
throw new IllegalArgumentException("Error: batch identifier supplied was not created by this class.");
}
MultiServerBatchIdentifier identifier = (MultiServerBatchIdentifier) batchIdentifier;
BatchDataContainer batchDataContainer = new BatchDataContainer(identifier, batchChunkList, startPosition);
// Create job
MultiServerPostBatchDataWorker worker =
new MultiServerPostBatchDataWorker(clients, minAbsoluteRedundancy, batchDataContainer, POST_MESSAGE_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
}
@Override
public void postBatchData(BatchIdentifier batchIdentifier, List<BatchChunk> batchChunkList, FutureCallback<Boolean> callback)
throws IllegalArgumentException {
postBatchData(batchIdentifier, batchChunkList, 0, callback);
}
@Override
public void closeBatch(BatchIdentifier payload, Timestamp timestamp, Iterable<Signature> signatures, FutureCallback<Boolean> callback)
throws IllegalArgumentException{
if (!(payload instanceof MultiServerBatchIdentifier)) {
throw new IllegalArgumentException("Error: batch identifier supplied was not created by this class.");
}
MultiServerBatchIdentifier identifier = (MultiServerBatchIdentifier) payload;
// Create job
MultiServerCloseBatchWorker worker =
new MultiServerCloseBatchWorker(clients, minAbsoluteRedundancy, identifier, timestamp, signatures, POST_MESSAGE_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
}
/**
@ -83,17 +197,16 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient {
* Return the number of databases in which the message was found
* Only try once per DB
* Ignore communication exceptions in specific databases
* @param id is the requested message ID
* @return the number of DBs in which retrieval was successful
*/
@Override
public void getRedundancy(MessageID id, ClientCallback<Float> callback) {
public void getRedundancy(MessageID id, FutureCallback<Float> callback) {
// Create job
BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.GET_REDUNDANCY, id, 1);
MultiServerGetRedundancyWorker worker =
new MultiServerGetRedundancyWorker(clients, minAbsoluteRedundancy, id, GET_REDUNDANCY_RETRY_NUM, callback);
// Submit job and create callback
Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new GetRedundancyFutureCallback(listeningExecutor, callback));
// Submit job
executorService.submit(worker);
}
@ -101,27 +214,69 @@ public class ThreadedBulletinBoardClient implements BulletinBoardClient {
* Go through the DBs and try to retrieve messages according to the specified filter
* If at the operation is successful for some DB: return the results and stop iterating
* If no operation is successful: return null (NOT blank list)
* @param filterList return only messages that match the filters (null means no filtering).
* @return
*/
@Override
public void readMessages(MessageFilterList filterList, ClientCallback<List<BulletinBoardMessage>> callback) {
public void readMessages(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
// Create job
BulletinClientJob job = new BulletinClientJob(meerkatDBs, minAbsoluteRedundancy, BulletinClientJob.JobType.READ_MESSAGES,
filterList, READ_MESSAGES_RETRY_NUM);
MultiServerReadMessagesWorker worker =
new MultiServerReadMessagesWorker(clients, minAbsoluteRedundancy, filterList, READ_MESSAGES_RETRY_NUM, callback);
// Submit job and create callback
Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), new ReadMessagesFutureCallback(listeningExecutor, callback));
// Submit job
executorService.submit(worker);
}
@Override
public void readMessage(MessageID msgID, FutureCallback<BulletinBoardMessage> callback) {
//Create job
MultiServerReadMessageWorker worker =
new MultiServerReadMessageWorker(clients, minAbsoluteRedundancy, msgID, READ_MESSAGES_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
}
@Override
public void readBatchData(BulletinBoardMessage stub, FutureCallback<BulletinBoardMessage> callback) throws IllegalArgumentException {
if (stub.getMsg().getDataTypeCase() != UnsignedBulletinBoardMessage.DataTypeCase.MSGID) {
throw new IllegalArgumentException("Message is not a stub and does not contain the required message ID");
}
// Create job
MultiServerReadBatchDataWorker worker =
new MultiServerReadBatchDataWorker(clients, minAbsoluteRedundancy, stub, READ_MESSAGES_RETRY_NUM, callback);
// Submit job
executorService.submit(worker);
}
/**
* This method is not supported by this class!
* This is because it has no meaning when considering more than one server without knowing which server will be contacted
*/
@Override
public void querySync(SyncQuery syncQuery, FutureCallback<SyncQueryResponse> callback) {
callback.onFailure(new IllegalAccessError("querySync is not supported by this class"));
}
@Override
public void close() {
super.close();
try {
listeningExecutor.shutdown();
while (! listeningExecutor.isShutdown()) {
listeningExecutor.awaitTermination(10, TimeUnit.SECONDS);
for (SingleServerBulletinBoardClient client : clients){
client.close();
}
executorService.shutdown();
while (! executorService.isShutdown()) {
executorService.awaitTermination(10, TimeUnit.SECONDS);
}
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());

276
bulletin-board-client/src/main/java/meerkat/bulletinboard/ThreadedBulletinBoardSubscriber.java Normal file
View File

@ -0,0 +1,276 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.Timestamp;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.util.BulletinBoardUtils;
import static meerkat.protobuf.BulletinBoardAPI.FilterType.*;
import java.util.*;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* Created by Arbel Deutsch Peled on 03-Mar-16.
* A multi-server implementation of the {@link BulletinBoardSubscriber}
*/
public class ThreadedBulletinBoardSubscriber implements BulletinBoardSubscriber {
protected final Collection<SubscriptionBulletinBoardClient> clients;
protected final BulletinBoardClient localClient;
protected Iterator<SubscriptionBulletinBoardClient> clientIterator;
protected SubscriptionBulletinBoardClient currentClient;
private long lastServerSwitchTime;
private AtomicBoolean isSyncInProgress;
private Semaphore rescheduleSemaphore;
private AtomicBoolean stopped;
private static final Float[] BREAKPOINTS = {0.5f, 0.75f, 0.9f, 0.95f, 0.99f, 0.999f};
public ThreadedBulletinBoardSubscriber(Collection<SubscriptionBulletinBoardClient> clients, BulletinBoardClient localClient) {
this.clients = clients;
this.localClient = localClient;
lastServerSwitchTime = System.currentTimeMillis();
clientIterator = clients.iterator();
currentClient = clientIterator.next();
isSyncInProgress = new AtomicBoolean(false);
rescheduleSemaphore = new Semaphore(1);
stopped = new AtomicBoolean(false);
}
/**
* Moves to next client and performs resync with it
*/
private void nextClient() {
try {
rescheduleSemaphore.acquire();
if (!clientIterator.hasNext()){
clientIterator = clients.iterator();
}
currentClient = clientIterator.next();
lastServerSwitchTime = System.currentTimeMillis();
isSyncInProgress.set(false);
rescheduleSemaphore.release();
} catch (InterruptedException e) {
// TODO: log
// Do not change client
}
}
private abstract class SubscriberCallback<T> implements FutureCallback<T> {
protected final MessageFilterList filterList;
protected final FutureCallback<List<BulletinBoardMessage>> callback;
private final long invocationTime;
public SubscriberCallback(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
this.filterList = filterList;
this.callback = callback;
this.invocationTime = System.currentTimeMillis();
}
/**
* Handles resyncing process for the given subscription after a server is switched
* Specifically: generates a sync query from the local database and uses it to query the current server
*/
private void reSync() {
SyncQuery syncQuery = null;
try {
syncQuery = localClient.generateSyncQuery(GenerateSyncQueryParams.newBuilder()
.setFilterList(filterList)
.addAllBreakpointList(Arrays.asList(BREAKPOINTS))
.build());
} catch (CommunicationException e) {
// Handle failure in standard way
onFailure(e);
}
currentClient.querySync(syncQuery, new SyncQueryCallback(filterList, callback));
}
/**
* Reschedules the subscription
*/
private void reschedule() {
try {
rescheduleSemaphore.acquire();
reSync();
rescheduleSemaphore.release();
} catch (InterruptedException e) {
//TODO: log
if (callback != null)
callback.onFailure(e); // Hard error: Cannot guarantee subscription safety
}
}
@Override
public void onFailure(Throwable t) {
// If server failure is not already known: switch to next client and resync
if (invocationTime > lastServerSwitchTime){
// Make sure only what thread switches the client
if (isSyncInProgress.compareAndSet(false, true)){
nextClient();
}
}
reschedule();
}
}
/**
* Provides handling logic for resync query callback operation
* Receives a SyncQueryResponse and reads the missing data (starting from the received timestamp) if needed
*/
protected class SyncQueryCallback extends SubscriberCallback<SyncQueryResponse> {
public SyncQueryCallback (MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
super(filterList, callback);
}
@Override
public void onSuccess(SyncQueryResponse result) {
final Timestamp DEFAULT_TIME = BulletinBoardUtils.toTimestampProto(946728000); // Year 2000
// Read required messages according to received Timestamp
Timestamp syncTimestamp;
if (result.hasLastTimeOfSync()) {
syncTimestamp = result.getLastTimeOfSync(); // Use returned time of sync
} else {
syncTimestamp = DEFAULT_TIME; // Get all messages
}
MessageFilterList timestampedFilterList = filterList.toBuilder()
.removeFilter(filterList.getFilterCount()-1) // Remove MIN_ENTRY filter
.addFilter(MessageFilter.newBuilder() // Add timestamp filter
.setType(AFTER_TIME)
.setTimestamp(syncTimestamp)
.build())
.build();
currentClient.readMessages(timestampedFilterList, new ReSyncCallback(filterList, callback, result.getLastEntryNum()));
}
}
/**
* Provides handling logic for callback of resyncing process
* Receives the missing messages, handles them and resubscribes
*/
protected class ReSyncCallback extends SubscriberCallback<List<BulletinBoardMessage>> {
private long minEntry;
public ReSyncCallback (MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback, long minEntry) {
super(filterList, callback);
this.minEntry = minEntry;
}
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
// Propagate result to caller
if (callback != null)
callback.onSuccess(result);
// Renew subscription
MessageFilterList newFilterList = filterList.toBuilder()
.removeFilter(filterList.getFilterCount()-1) // Remove current MIN_ENTRY filter
.addFilter(MessageFilter.newBuilder() // Add new MIN_ENTRY filter for current server
.setType(MIN_ENTRY)
.setEntry(minEntry)
.build())
.build();
currentClient.subscribe(newFilterList, callback);
}
}
/**
* Provides the handling logic for results and failures of main subscription (while there are no errors)
*/
protected class SubscriptionCallback extends SubscriberCallback<List<BulletinBoardMessage>> {
public SubscriptionCallback(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback){
super(filterList, callback);
}
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
// Propagate result to caller
if (callback != null)
callback.onSuccess(result);
}
}
@Override
public void subscribe(MessageFilterList filterList, long startEntry, FutureCallback<List<BulletinBoardMessage>> callback) {
currentClient.subscribe(filterList, startEntry, new SubscriptionCallback(filterList, callback));
}
@Override
public void subscribe(MessageFilterList filterList, FutureCallback<List<BulletinBoardMessage>> callback) {
subscribe(filterList, 0, callback);
}
}

25
bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ClientFutureCallback.java
View File

@ -1,25 +0,0 @@
package meerkat.bulletinboard.callbacks;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListeningExecutorService;
import meerkat.bulletinboard.BulletinClientJob;
import meerkat.bulletinboard.BulletinClientJobResult;
import meerkat.bulletinboard.BulletinClientWorker;
import meerkat.protobuf.BulletinBoardAPI;
import java.util.List;
/**
* This is a future callback used to listen to workers and run on job finish
* Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error
*/
public abstract class ClientFutureCallback implements FutureCallback<BulletinClientJobResult> {
protected ListeningExecutorService listeningExecutor;
ClientFutureCallback(ListeningExecutorService listeningExecutor) {
this.listeningExecutor = listeningExecutor;
}
}

38
bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/GetRedundancyFutureCallback.java
View File

@ -1,38 +0,0 @@
package meerkat.bulletinboard.callbacks;
import com.google.common.util.concurrent.ListeningExecutorService;
import meerkat.bulletinboard.BulletinBoardClient;
import meerkat.bulletinboard.BulletinClientJobResult;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.List;
/**
* This is a future callback used to listen to workers and run on job finish
* Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error
*/
public class GetRedundancyFutureCallback extends ClientFutureCallback {
private BulletinBoardClient.ClientCallback<Float> callback;
public GetRedundancyFutureCallback(ListeningExecutorService listeningExecutor,
BulletinBoardClient.ClientCallback<Float> callback) {
super(listeningExecutor);
this.callback = callback;
}
@Override
public void onSuccess(BulletinClientJobResult result) {
int absoluteRedundancy = ((IntMsg) result.getResult()).getValue();
int totalServers = result.getJob().getServerAddresses().size();
callback.handleCallback( ((float) absoluteRedundancy) / ((float) totalServers) );
}
@Override
public void onFailure(Throwable t) {
callback.handleFailure(t);
}
}

46
bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/PostMessageFutureCallback.java
View File

@ -1,46 +0,0 @@
package meerkat.bulletinboard.callbacks;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListeningExecutorService;
import meerkat.bulletinboard.BulletinBoardClient;
import meerkat.bulletinboard.BulletinClientJob;
import meerkat.bulletinboard.BulletinClientJobResult;
import meerkat.bulletinboard.BulletinClientWorker;
import meerkat.protobuf.BulletinBoardAPI;
import java.util.List;
/**
* This is a future callback used to listen to workers and run on job finish
* Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error
*/
public class PostMessageFutureCallback extends ClientFutureCallback {
private BulletinBoardClient.ClientCallback<?> callback;
public PostMessageFutureCallback(ListeningExecutorService listeningExecutor,
BulletinBoardClient.ClientCallback<?> callback) {
super(listeningExecutor);
this.callback = callback;
}
@Override
public void onSuccess(BulletinClientJobResult result) {
BulletinClientJob job = result.getJob();
job.decMaxRetry();
// If redundancy is below threshold: retry
if (job.getMinServers() > 0 && job.isRetry()) {
Futures.addCallback(listeningExecutor.submit(new BulletinClientWorker(job)), this);
}
callback.handleCallback(null);
}
@Override
public void onFailure(Throwable t) {
callback.handleFailure(t);
}
}

38
bulletin-board-client/src/main/java/meerkat/bulletinboard/callbacks/ReadMessagesFutureCallback.java
View File

@ -1,38 +0,0 @@
package meerkat.bulletinboard.callbacks;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListeningExecutorService;
import meerkat.bulletinboard.BulletinBoardClient;
import meerkat.bulletinboard.BulletinClientJob;
import meerkat.bulletinboard.BulletinClientJobResult;
import meerkat.bulletinboard.BulletinClientWorker;
import meerkat.protobuf.BulletinBoardAPI;
import java.util.List;
/**
* This is a future callback used to listen to workers and run on job finish
* Depending on the type of job and the finishing status of the worker: a decision is made whether to retry or return an error
*/
public class ReadMessagesFutureCallback extends ClientFutureCallback {
private BulletinBoardClient.ClientCallback<List<BulletinBoardAPI.BulletinBoardMessage>> callback;
public ReadMessagesFutureCallback(ListeningExecutorService listeningExecutor,
BulletinBoardClient.ClientCallback<List<BulletinBoardAPI.BulletinBoardMessage>> callback) {
super(listeningExecutor);
this.callback = callback;
}
@Override
public void onSuccess(BulletinClientJobResult result) {
callback.handleCallback(((BulletinBoardAPI.BulletinBoardMessageList) result.getResult()).getMessageList());
}
@Override
public void onFailure(Throwable t) {
callback.handleFailure(t);
}
}

96
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerBeginBatchWorker.java Normal file
View File

@ -0,0 +1,96 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.MultiServerBatchIdentifier;
import meerkat.bulletinboard.MultiServerWorker;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.bulletinboard.AsyncBulletinBoardClient.BatchIdentifier;
import meerkat.comm.CommunicationException;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerBeginBatchWorker extends MultiServerWorker<Iterable<String>, BatchIdentifier> {
private BatchIdentifier[] identifiers;
private AtomicInteger remainingServers;
public MultiServerBeginBatchWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, Iterable<String> payload, int maxRetry,
FutureCallback<BatchIdentifier> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
identifiers = new BatchIdentifier[clients.size()];
for (int i = 0 ; i < identifiers.length ; i++) {
identifiers[i] = null;
}
remainingServers = new AtomicInteger(clients.size());
}
private class BeginBatchCallback implements FutureCallback<BatchIdentifier> {
private final int clientNum;
public BeginBatchCallback(int clientNum) {
this.clientNum = clientNum;
}
private void finishPost() {
if (remainingServers.decrementAndGet() <= 0){
if (minServers.decrementAndGet() <= 0) {
MultiServerBeginBatchWorker.this.onSuccess(new MultiServerBatchIdentifier(identifiers));
} else {
MultiServerBeginBatchWorker.this.onFailure(new CommunicationException("Could not open batch in enough servers"));
}
}
}
@Override
public void onSuccess(BatchIdentifier result) {
identifiers[clientNum] = result;
finishPost();
}
@Override
public void onFailure(Throwable t) {
finishPost();
}
}
@Override
public void onSuccess(BatchIdentifier result) {
succeed(result);
}
@Override
public void onFailure(Throwable t) {
fail(t);
}
@Override
public void run() {
int clientNum = 0;
for (SingleServerBulletinBoardClient client : clients){
client.beginBatch(payload, new BeginBatchCallback(clientNum));
clientNum++;
}
}
}

84
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerCloseBatchWorker.java Normal file
View File

@ -0,0 +1,84 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.Timestamp;
import meerkat.bulletinboard.AsyncBulletinBoardClient.BatchIdentifier;
import meerkat.bulletinboard.BatchDataContainer;
import meerkat.bulletinboard.MultiServerBatchIdentifier;
import meerkat.bulletinboard.MultiServerWorker;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.crypto.DigitalSignature;
import meerkat.protobuf.Crypto;
import meerkat.protobuf.Crypto.Signature;
import java.util.Iterator;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerCloseBatchWorker extends MultiServerWorker<MultiServerBatchIdentifier, Boolean> {
private final Timestamp timestamp;
private final Iterable<Crypto.Signature> signatures;
public MultiServerCloseBatchWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, MultiServerBatchIdentifier payload, Timestamp timestamp, Iterable<Crypto.Signature> signatures,
int maxRetry, FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
this.timestamp = timestamp;
this.signatures = signatures;
}
@Override
public void run() {
Iterator<BatchIdentifier> identifierIterator = payload.getIdentifiers().iterator();
// Iterate through client
for (SingleServerBulletinBoardClient client : clients) {
if (identifierIterator.hasNext()) {
// Fetch the batch identifier supplied by the specific client (may be null if batch open failed on client
BatchIdentifier identifier = identifierIterator.next();
if (identifier != null) {
// Post the data with the matching identifier to the client
client.closeBatch(identifier, timestamp, signatures, this);
} else {
// Count servers with no batch identifier as failed
maxFailedServers.decrementAndGet();
}
}
}
}
@Override
public void onSuccess(Boolean result) {
if (minServers.decrementAndGet() <= 0){
succeed(result);
}
}
@Override
public void onFailure(Throwable t) {
if (maxFailedServers.decrementAndGet() <= 0){
fail(t);
}
}
}

62
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericPostWorker.java Normal file
View File

@ -0,0 +1,62 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.MultiServerWorker;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.comm.CommunicationException;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import java.util.Iterator;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public abstract class MultiServerGenericPostWorker<T> extends MultiServerWorker<T, Boolean> {
public MultiServerGenericPostWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, T payload, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
protected abstract void doPost(SingleServerBulletinBoardClient client, T payload);
/**
* This method carries out the actual communication with the servers via HTTP Post
* It accesses the servers one by one and tries to post the payload to each in turn
* The method will only iterate once through the server list
* Successful post to a server results in removing the server from the list
*/
public void run() {
// Iterate through servers
for (SingleServerBulletinBoardClient client : clients) {
// Send request to Server
doPost(client, payload);
}
}
@Override
public void onSuccess(Boolean result) {
if (result){
if (minServers.decrementAndGet() <= 0){
succeed(Boolean.TRUE);
}
}
}
@Override
public void onFailure(Throwable t) {
if (maxFailedServers.decrementAndGet() < 0){
fail(t);
}
}
}

69
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGenericReadWorker.java Normal file
View File

@ -0,0 +1,69 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.MultiServerWorker;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.comm.CommunicationException;
import java.util.Iterator;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public abstract class MultiServerGenericReadWorker<IN, OUT> extends MultiServerWorker<IN, OUT>{
private Iterator<SingleServerBulletinBoardClient> clientIterator;
private String errorString;
public MultiServerGenericReadWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, IN payload, int maxRetry,
FutureCallback<OUT> futureCallback) {
super(clients, true, minServers, payload, maxRetry, futureCallback); // Shuffle clients on creation to balance load
clientIterator = clients.iterator();
errorString = "";
}
protected abstract void doRead(IN payload, SingleServerBulletinBoardClient client);
/**
* This method carries out the actual communication with the servers via HTTP Post
* It accesses the servers in a random order until one answers it
* Successful retrieval from any server terminates the method and returns the received values; The list is not changed
*/
public void run(){
// Iterate through servers
if (clientIterator.hasNext()) {
// Get next server
SingleServerBulletinBoardClient client = clientIterator.next();
// Retrieve answer from server
doRead(payload, client);
} else {
fail(new CommunicationException("Could not contact any server. Errors follow:\n" + errorString));
}
}
@Override
public void onSuccess(OUT msg) {
succeed(msg);
}
@Override
public void onFailure(Throwable t) {
//TODO: log
errorString += t.getCause() + " " + t.getMessage() + "\n";
run(); // Retry with next server
}
}

67
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerGetRedundancyWorker.java Normal file
View File

@ -0,0 +1,67 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.MultiServerWorker;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerGetRedundancyWorker extends MultiServerWorker<MessageID, Float> {
private AtomicInteger serversContainingMessage;
private AtomicInteger totalContactedServers;
public MultiServerGetRedundancyWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, MessageID payload, int maxRetry,
FutureCallback<Float> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback); // Shuffle clients on creation to balance load
serversContainingMessage = new AtomicInteger(0);
totalContactedServers = new AtomicInteger(0);
}
/**
* This method carries out the actual communication with the servers via HTTP Post
* It accesses the servers in a random order until one answers it
* Successful retrieval from any server terminates the method and returns the received values; The list is not changed
*/
public void run(){
// Iterate through clients
for (SingleServerBulletinBoardClient client : clients) {
// Send request to client
client.getRedundancy(payload,this);
}
}
@Override
public void onSuccess(Float result) {
if (result > 0.5) {
serversContainingMessage.incrementAndGet();
}
if (totalContactedServers.incrementAndGet() >= getClientNumber()){
succeed(((float) serversContainingMessage.get()) / ((float) getClientNumber()));
}
}
@Override
public void onFailure(Throwable t) {
onSuccess(0.0f);
}
}

72
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchDataWorker.java Normal file
View File

@ -0,0 +1,72 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.AsyncBulletinBoardClient.BatchIdentifier;
import meerkat.bulletinboard.MultiServerWorker;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.bulletinboard.BatchDataContainer;
import java.util.Iterator;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerPostBatchDataWorker extends MultiServerWorker<BatchDataContainer, Boolean> {
public MultiServerPostBatchDataWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, BatchDataContainer payload, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
public void run() {
Iterator<BatchIdentifier> identifierIterator = payload.batchId.getIdentifiers().iterator();
// Iterate through client
for (SingleServerBulletinBoardClient client : clients) {
if (identifierIterator.hasNext()) {
// Fetch the batch identifier supplied by the specific client (may be null if batch open failed on client
BatchIdentifier identifier = identifierIterator.next();
if (identifier != null) {
// Post the data with the matching identifier to the client
client.postBatchData(identifier, payload.batchChunkList, payload.startPosition, this);
} else {
// Count servers with no batch identifier as failed
maxFailedServers.decrementAndGet();
}
}
}
}
@Override
public void onSuccess(Boolean result) {
if (minServers.decrementAndGet() <= 0){
succeed(result);
}
}
@Override
public void onFailure(Throwable t) {
if (maxFailedServers.decrementAndGet() <= 0){
fail(t);
}
}
}

34
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostBatchWorker.java Normal file
View File

@ -0,0 +1,34 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerPostBatchWorker extends MultiServerGenericPostWorker<BulletinBoardMessage> {
private final int chunkSize;
public MultiServerPostBatchWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, BulletinBoardMessage payload, int chunkSize, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
this.chunkSize = chunkSize;
}
@Override
protected void doPost(SingleServerBulletinBoardClient client, BulletinBoardMessage payload) {
client.postAsBatch(payload, chunkSize, this);
}
}

28
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerPostMessageWorker.java Normal file
View File

@ -0,0 +1,28 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerPostMessageWorker extends MultiServerGenericPostWorker<BulletinBoardMessage> {
public MultiServerPostMessageWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, BulletinBoardMessage payload, int maxRetry,
FutureCallback<Boolean> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doPost(SingleServerBulletinBoardClient client, BulletinBoardMessage payload) {
client.postMessage(payload, this);
}
}

29
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadBatchDataWorker.java Normal file
View File

@ -0,0 +1,29 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerReadBatchDataWorker extends MultiServerGenericReadWorker<BulletinBoardMessage, BulletinBoardMessage> {
public MultiServerReadBatchDataWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, BulletinBoardMessage payload, int maxRetry,
FutureCallback<BulletinBoardMessage> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doRead(BulletinBoardMessage payload, SingleServerBulletinBoardClient client) {
client.readBatchData(payload, this);
}
}

30
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessageWorker.java Normal file
View File

@ -0,0 +1,30 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.protobuf.BulletinBoardAPI.MessageID;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerReadMessageWorker extends MultiServerGenericReadWorker<MessageID, BulletinBoardMessage> {
public MultiServerReadMessageWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, MessageID payload, int maxRetry,
FutureCallback<BulletinBoardMessage> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doRead(MessageID payload, SingleServerBulletinBoardClient client) {
client.readMessage(payload, this);
}
}

29
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/multiserver/MultiServerReadMessagesWorker.java Normal file
View File

@ -0,0 +1,29 @@
package meerkat.bulletinboard.workers.multiserver;
import com.google.common.util.concurrent.FutureCallback;
import meerkat.bulletinboard.SingleServerBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.*;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class MultiServerReadMessagesWorker extends MultiServerGenericReadWorker<MessageFilterList,List<BulletinBoardMessage>>{
public MultiServerReadMessagesWorker(List<SingleServerBulletinBoardClient> clients,
int minServers, MessageFilterList payload, int maxRetry,
FutureCallback<List<BulletinBoardMessage>> futureCallback) {
super(clients, minServers, payload, maxRetry, futureCallback);
}
@Override
protected void doRead(MessageFilterList payload, SingleServerBulletinBoardClient client) {
client.readMessages(payload, this);
}
}

53
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerBeginBatchWorker.java Normal file
View File

@ -0,0 +1,53 @@
package meerkat.bulletinboard.workers.singleserver;
import com.google.protobuf.Int64Value;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage;
import meerkat.rest.Constants;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import static meerkat.bulletinboard.BulletinBoardConstants.BEGIN_BATCH_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a post operation
*/
public class SingleServerBeginBatchWorker extends SingleServerWorker<BeginBatchMessage,Int64Value> {
public SingleServerBeginBatchWorker(String serverAddress, BeginBatchMessage payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
}
@Override
public Int64Value call() throws Exception {
Client client = clientLocal.get();
WebTarget webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(BEGIN_BATCH_PATH);
Response response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(
Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF));
try {
Int64Value result = response.readEntity(Int64Value.class);
return result;
} catch (ProcessingException | IllegalStateException e) {
// Post to this server failed
throw new CommunicationException("Could not contact the server. Original error: " + e.getMessage());
} catch (Exception e) {
throw new CommunicationException("Could not contact the server. Original error: " + e.getMessage());
}
finally {
response.close();
}
}
}

17
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerCloseBatchWorker.java Normal file
View File

@ -0,0 +1,17 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.protobuf.BulletinBoardAPI.CloseBatchMessage;
import static meerkat.bulletinboard.BulletinBoardConstants.CLOSE_BATCH_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a stop batch operation
*/
public class SingleServerCloseBatchWorker extends SingleServerGenericPostWorker<CloseBatchMessage> {
public SingleServerCloseBatchWorker(String serverAddress, CloseBatchMessage payload, int maxRetry) {
super(serverAddress, CLOSE_BATCH_PATH, payload, maxRetry);
}
}

55
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenerateSyncQueryWorker.java Normal file
View File

@ -0,0 +1,55 @@
package meerkat.bulletinboard.workers.singleserver;
import com.google.protobuf.Int64Value;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.SyncQuery;
import meerkat.protobuf.BulletinBoardAPI.GenerateSyncQueryParams;
import meerkat.rest.Constants;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.GENERATE_SYNC_QUERY_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a Sync Query Generation operation
*/
public class SingleServerGenerateSyncQueryWorker extends SingleServerWorker<GenerateSyncQueryParams,SyncQuery> {
public SingleServerGenerateSyncQueryWorker(String serverAddress, GenerateSyncQueryParams payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
}
@Override
public SyncQuery call() throws Exception {
Client client = clientLocal.get();
WebTarget webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(GENERATE_SYNC_QUERY_PATH);
Response response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF));
try {
SyncQuery result = response.readEntity(SyncQuery.class);
return result;
} catch (ProcessingException | IllegalStateException e) {
// Post to this server failed
throw new CommunicationException("Could not contact the server. Original error: " + e.getMessage());
} catch (Exception e) {
throw new CommunicationException("Could not contact the server. Original error: " + e.getMessage());
}
finally {
response.close();
}
}
}

63
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGenericPostWorker.java Normal file
View File

@ -0,0 +1,63 @@
package meerkat.bulletinboard.workers.singleserver;
import com.google.protobuf.BoolValue;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.Comm.*;
import meerkat.rest.Constants;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a post operation
*/
public class SingleServerGenericPostWorker<T> extends SingleServerWorker<T, Boolean> {
private final String subPath;
public SingleServerGenericPostWorker(String serverAddress, String subPath, T payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
this.subPath = subPath;
}
/**
* This method carries out the actual communication with the server via HTTP Post
* It accesses the server and tries to post the payload to it
* Successful post to a server results
* @return TRUE if the operation is successful
* @throws CommunicationException if the operation is unsuccessful
*/
public Boolean call() throws CommunicationException{
Client client = clientLocal.get();
WebTarget webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(subPath);
Response response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(
Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF));
try {
// If a BoolValue entity is returned: the post was successful
response.readEntity(BoolValue.class);
return Boolean.TRUE;
} catch (ProcessingException | IllegalStateException e) {
// Post to this server failed
throw new CommunicationException("Could not contact the server");
}
finally {
response.close();
}
}
}

85
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerGetRedundancyWorker.java Normal file
View File

@ -0,0 +1,85 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageInputStream;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.rest.Constants;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class SingleServerGetRedundancyWorker extends SingleServerWorker<MessageID, Float> {
public SingleServerGetRedundancyWorker(String serverAddress, MessageID payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
}
/**
* This method carries out the actual communication with the server via HTTP Post
* It queries the server for a message with the given ID
* @return TRUE if the message exists in the server and FALSE otherwise
* @throws CommunicationException if the server does not return a valid answer
*/
public Float call() throws CommunicationException{
Client client = clientLocal.get();
WebTarget webTarget;
Response response;
MessageFilterList msgFilterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(payload.getID())
.build()
).build();
// Send request to Server
// Send request to Server
webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH);
InputStream in = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msgFilterList, Constants.MEDIATYPE_PROTOBUF), InputStream.class);
MessageInputStream<BulletinBoardMessage> inputStream = null;
// Retrieve answer
try {
inputStream = MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BulletinBoardMessage.class);
if (inputStream.asList().size() > 0){
// Message exists in the server
return 1.0f;
}
else {
// Message does not exist in the server
return 0.0f;
}
} catch (Exception e) {
// Read failed
throw new CommunicationException("Server access failed");
} finally {
try {
inputStream.close();
} catch (IOException ignored) {}
}
}
}

17
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostBatchWorker.java Normal file
View File

@ -0,0 +1,17 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.protobuf.BulletinBoardAPI.BatchMessage;
import static meerkat.bulletinboard.BulletinBoardConstants.POST_BATCH_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a post batch operation
*/
public class SingleServerPostBatchWorker extends SingleServerGenericPostWorker<BatchMessage> {
public SingleServerPostBatchWorker(String serverAddress, BatchMessage payload, int maxRetry) {
super(serverAddress, POST_BATCH_PATH, payload, maxRetry);
}
}

17
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerPostMessageWorker.java Normal file
View File

@ -0,0 +1,17 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import static meerkat.bulletinboard.BulletinBoardConstants.POST_MESSAGE_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a post operation
*/
public class SingleServerPostMessageWorker extends SingleServerGenericPostWorker<BulletinBoardMessage> {
public SingleServerPostMessageWorker(String serverAddress, BulletinBoardMessage payload, int maxRetry) {
super(serverAddress, POST_MESSAGE_PATH, payload, maxRetry);
}
}

59
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerQuerySyncWorker.java Normal file
View File

@ -0,0 +1,59 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.SyncQuery;
import meerkat.protobuf.BulletinBoardAPI.SyncQueryResponse;
import meerkat.rest.Constants;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.SYNC_QUERY_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
* Tries to contact server once and perform a post operation
*/
public class SingleServerQuerySyncWorker extends SingleServerWorker<SyncQuery, SyncQueryResponse> {
public SingleServerQuerySyncWorker(String serverAddress, SyncQuery payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
}
@Override
public SyncQueryResponse call() throws Exception {
Client client = clientLocal.get();
WebTarget webTarget;
Response response;
// Send request to Server
webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(SYNC_QUERY_PATH);
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF));
// Retrieve answer
try {
// If a BulletinBoardMessageList is returned: the read was successful
return response.readEntity(SyncQueryResponse.class);
} catch (ProcessingException | IllegalStateException e) {
// Read failed
throw new CommunicationException("Server access failed");
}
finally {
response.close();
}
}
}

71
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadBatchWorker.java Normal file
View File

@ -0,0 +1,71 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageInputStream;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.rest.Constants;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.READ_BATCH_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class SingleServerReadBatchWorker extends SingleServerWorker<BatchQuery, List<BatchChunk>> {
public SingleServerReadBatchWorker(String serverAddress, BatchQuery payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
}
/**
* This method carries out the actual communication with the server via HTTP Post
* Upon successful retrieval from the server the method returns the received values
* @return the complete batch as read from the server
* @throws CommunicationException if the server's response is invalid
*/
public List<BatchChunk> call() throws CommunicationException{
Client client = clientLocal.get();
WebTarget webTarget;
// Get the batch data
webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_BATCH_PATH);
InputStream in = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF), InputStream.class);
MessageInputStream<BatchChunk> inputStream = null;
try {
inputStream = MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BatchChunk.class);
return inputStream.asList();
} catch (IOException | InvocationTargetException e) {
// Read failed
throw new CommunicationException("Could not contact the server or server returned illegal result");
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new CommunicationException("MessageInputStream error");
} finally {
try {
inputStream.close();
} catch (IOException ignored) {}
}
}
}

76
bulletin-board-client/src/main/java/meerkat/bulletinboard/workers/singleserver/SingleServerReadMessagesWorker.java Normal file
View File

@ -0,0 +1,76 @@
package meerkat.bulletinboard.workers.singleserver;
import meerkat.bulletinboard.SingleServerWorker;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageInputStream;
import meerkat.protobuf.BulletinBoardAPI;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessageList;
import meerkat.protobuf.BulletinBoardAPI.MessageFilterList;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.rest.Constants;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import static meerkat.bulletinboard.BulletinBoardConstants.BULLETIN_BOARD_SERVER_PATH;
import static meerkat.bulletinboard.BulletinBoardConstants.READ_MESSAGES_PATH;
/**
* Created by Arbel Deutsch Peled on 27-Dec-15.
*/
public class SingleServerReadMessagesWorker extends SingleServerWorker<MessageFilterList, List<BulletinBoardMessage>> {
public SingleServerReadMessagesWorker(String serverAddress, MessageFilterList payload, int maxRetry) {
super(serverAddress, payload, maxRetry);
}
/**
* This method carries out the actual communication with the server via HTTP Post
* Upon successful retrieval from the server the method returns the received values
* @return The list of messages returned by the server
* @throws CommunicationException if the server's response is invalid
*/
public List<BulletinBoardMessage> call() throws CommunicationException{
Client client = clientLocal.get();
WebTarget webTarget;
// Send request to Server
webTarget = client.target(serverAddress).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH);
InputStream in = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(payload, Constants.MEDIATYPE_PROTOBUF), InputStream.class);
MessageInputStream<BulletinBoardMessage> inputStream = null;
try {
inputStream = MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BulletinBoardMessage.class);
return inputStream.asList();
} catch (IOException | InvocationTargetException e) {
// Read failed
throw new CommunicationException("Could not contact the server or server returned illegal result");
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new CommunicationException("MessageInputStream error");
} finally {
try {
inputStream.close();
} catch (IOException ignored) {}
}
}
}

214
bulletin-board-client/src/test/java/BulletinBoardClientIntegrationTest.java
View File

@ -1,214 +0,0 @@
import com.google.protobuf.ByteString;
import meerkat.bulletinboard.BulletinBoardClient;
import meerkat.bulletinboard.BulletinBoardClient.ClientCallback;
import meerkat.bulletinboard.ThreadedBulletinBoardClient;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto;
import meerkat.protobuf.Voting.*;
import meerkat.util.BulletinBoardMessageComparator;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.number.OrderingComparison.*;
import java.util.*;
import java.util.concurrent.Semaphore;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
*/
public class BulletinBoardClientIntegrationTest {
Semaphore jobSemaphore;
Vector<Throwable> thrown;
private class PostCallback implements ClientCallback<Object>{
@Override
public void handleCallback(Object msg) {
System.err.println("Post operation completed");
jobSemaphore.release();
}
@Override
public void handleFailure(Throwable t) {
thrown.add(t);
jobSemaphore.release();
}
}
private class RedundancyCallback implements ClientCallback<Float>{
private float minRedundancy;
public RedundancyCallback(float minRedundancy) {
this.minRedundancy = minRedundancy;
}
@Override
public void handleCallback(Float redundancy) {
System.err.println("Redundancy found is: " + redundancy);
jobSemaphore.release();
assertThat(redundancy, greaterThanOrEqualTo(minRedundancy));
}
@Override
public void handleFailure(Throwable t) {
thrown.add(t);
jobSemaphore.release();
}
}
private class ReadCallback implements ClientCallback<List<BulletinBoardMessage>>{
private List<BulletinBoardMessage> expectedMsgList;
public ReadCallback(List<BulletinBoardMessage> expectedMsgList) {
this.expectedMsgList = expectedMsgList;
}
@Override
public void handleCallback(List<BulletinBoardMessage> messages) {
System.err.println(messages);
jobSemaphore.release();
BulletinBoardMessageComparator msgComparator = new BulletinBoardMessageComparator();
assertThat(messages.size(), is(expectedMsgList.size()));
Iterator<BulletinBoardMessage> expectedMessageIterator = expectedMsgList.iterator();
Iterator<BulletinBoardMessage> receivedMessageIterator = messages.iterator();
while (expectedMessageIterator.hasNext()) {
assertThat(msgComparator.compare(expectedMessageIterator.next(), receivedMessageIterator.next()), is(0));
}
}
@Override
public void handleFailure(Throwable t) {
thrown.add(t);
jobSemaphore.release();
}
}
private BulletinBoardClient bulletinBoardClient;
private PostCallback postCallback;
private RedundancyCallback redundancyCallback;
private ReadCallback readCallback;
private static String PROP_GETTY_URL = "gretty.httpBaseURI";
private static String DEFAULT_BASE_URL = "http://localhost:8081";
private static String BASE_URL = System.getProperty(PROP_GETTY_URL, DEFAULT_BASE_URL);
@Before
public void init(){
bulletinBoardClient = new ThreadedBulletinBoardClient();
List<String> testDB = new LinkedList<String>();
testDB.add(BASE_URL);
bulletinBoardClient.init(BulletinBoardClientParams.newBuilder()
.addBulletinBoardAddress("http://localhost:8081")
.setMinRedundancy((float) 1.0)
.build());
postCallback = new PostCallback();
redundancyCallback = new RedundancyCallback((float) 1.0);
thrown = new Vector<>();
jobSemaphore = new Semaphore(0);
}
@Test
public void postTest() {
byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4};
byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14};
byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24};
byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0};
BulletinBoardMessage msg;
MessageFilterList filterList;
List<BulletinBoardMessage> msgList;
MessageID messageID;
Comparator<BulletinBoardMessage> msgComparator = new BulletinBoardMessageComparator();
msg = BulletinBoardMessage.newBuilder()
.setMsg(UnsignedBulletinBoardMessage.newBuilder()
.addTag("Signature")
.addTag("Trustee")
.setData(ByteString.copyFrom(b1))
.build())
.addSig(Crypto.Signature.newBuilder()
.setType(Crypto.SignatureType.DSA)
.setData(ByteString.copyFrom(b2))
.setSignerId(ByteString.copyFrom(b3))
.build())
.addSig(Crypto.Signature.newBuilder()
.setType(Crypto.SignatureType.ECDSA)
.setData(ByteString.copyFrom(b3))
.setSignerId(ByteString.copyFrom(b2))
.build())
.build();
messageID = bulletinBoardClient.postMessage(msg,postCallback);
try {
jobSemaphore.acquire();
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());
}
bulletinBoardClient.getRedundancy(messageID,redundancyCallback);
filterList = MessageFilterList.newBuilder()
.addFilter(
MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag("Signature")
.build()
)
.addFilter(
MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag("Trustee")
.build()
)
.build();
msgList = new LinkedList<BulletinBoardMessage>();
msgList.add(msg);
readCallback = new ReadCallback(msgList);
bulletinBoardClient.readMessages(filterList, readCallback);
try {
jobSemaphore.acquire(2);
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());
}
bulletinBoardClient.close();
for (Throwable t : thrown) {
System.err.println(t.getMessage());
}
if (thrown.size() > 0) {
assert false;
}
}
}

318
bulletin-board-client/src/test/java/meerkat/bulletinboard/BulletinBoardSynchronizerTest.java Normal file
View File

@ -0,0 +1,318 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.*;
import com.google.protobuf.Timestamp;
import static meerkat.bulletinboard.BulletinBoardSynchronizer.SyncStatus;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer;
import meerkat.bulletinboard.sqlserver.H2QueryProvider;
import meerkat.comm.CommunicationException;
import meerkat.crypto.concrete.ECDSASignature;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.util.BulletinBoardMessageComparator;
import meerkat.util.BulletinBoardMessageGenerator;
import org.junit.*;
import java.io.IOException;
import java.io.InputStream;
import java.security.*;
import java.security.cert.CertificateException;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.fail;
/**
* Created by Arbel on 6/1/2016.
*/
public class BulletinBoardSynchronizerTest {
private static final String REMOTE_SERVER_ADDRESS = "remoteDB";
private static final String LOCAL_SERVER_ADDRESS = "localDB";
private static int testCount;
private static final int THREAD_NUM = 3;
private static final int SUBSCRIPTION_INTERVAL = 1000;
private static final int SYNC_SLEEP_INTERVAL = 500;
private static final int SYNC_WAIT_CAP = 1000;
private DeletableSubscriptionBulletinBoardClient localClient;
private AsyncBulletinBoardClient remoteClient;
private BulletinBoardSynchronizer synchronizer;
private static BulletinBoardMessageGenerator messageGenerator;
private static BulletinBoardMessageComparator messageComparator;
private static String KEYFILE_EXAMPLE = "/certs/enduser-certs/user1-key-with-password-secret.p12";
private static String KEYFILE_PASSWORD1 = "secret";
private static String CERT1_PEM_EXAMPLE = "/certs/enduser-certs/user1.crt";
private static BulletinBoardSignature[] signers;
private static ByteString[] signerIDs;
private Semaphore semaphore;
private List<Throwable> thrown;
@BeforeClass
public static void build() {
messageGenerator = new BulletinBoardMessageGenerator(new Random(0));
messageComparator = new BulletinBoardMessageComparator();
signers = new BulletinBoardSignature[1];
signerIDs = new ByteString[1];
signers[0] = new GenericBulletinBoardSignature(new ECDSASignature());
signerIDs[0] = signers[0].getSignerID();
InputStream keyStream = BulletinBoardSynchronizerTest.class.getResourceAsStream(KEYFILE_EXAMPLE);
char[] password = KEYFILE_PASSWORD1.toCharArray();
try {
KeyStore.Builder keyStoreBuilder = signers[0].getPKCS12KeyStoreBuilder(keyStream, password);
signers[0].loadSigningCertificate(keyStoreBuilder);
signers[0].loadVerificationCertificates(BulletinBoardSynchronizerTest.class.getResourceAsStream(CERT1_PEM_EXAMPLE));
} catch (IOException e) {
System.err.println("Failed reading from signature file " + e.getMessage());
fail("Failed reading from signature file " + e.getMessage());
} catch (CertificateException e) {
System.err.println("Failed reading certificate " + e.getMessage());
fail("Failed reading certificate " + e.getMessage());
} catch (KeyStoreException e) {
System.err.println("Failed reading keystore " + e.getMessage());
fail("Failed reading keystore " + e.getMessage());
} catch (NoSuchAlgorithmException e) {
System.err.println("Couldn't find signing algorithm " + e.getMessage());
fail("Couldn't find signing algorithm " + e.getMessage());
} catch (UnrecoverableKeyException e) {
System.err.println("Couldn't find signing key " + e.getMessage());
fail("Couldn't find signing key " + e.getMessage());
}
signerIDs[0] = signers[0].getSignerID();
testCount = 0;
}
@Before
public void init() throws CommunicationException {
DeletableBulletinBoardServer remoteServer = new BulletinBoardSQLServer(new H2QueryProvider(REMOTE_SERVER_ADDRESS + testCount));
remoteServer.init();
remoteClient = new LocalBulletinBoardClient(
remoteServer,
THREAD_NUM,
SUBSCRIPTION_INTERVAL);
DeletableBulletinBoardServer localServer = new BulletinBoardSQLServer(new H2QueryProvider(LOCAL_SERVER_ADDRESS + testCount));
localServer.init();
localClient = new LocalBulletinBoardClient(
localServer,
THREAD_NUM,
SUBSCRIPTION_INTERVAL);
synchronizer = new SimpleBulletinBoardSynchronizer(SYNC_SLEEP_INTERVAL, SYNC_WAIT_CAP);
synchronizer.init(localClient, remoteClient);
semaphore = new Semaphore(0);
thrown = new LinkedList<>();
testCount++;
}
private class SyncStatusCallback implements FutureCallback<SyncStatus> {
private final SyncStatus statusToWaitFor;
private AtomicBoolean stillWaiting;
public SyncStatusCallback(SyncStatus statusToWaitFor) {
this.statusToWaitFor = statusToWaitFor;
stillWaiting = new AtomicBoolean(true);
}
@Override
public void onSuccess(SyncStatus result) {
if (result == statusToWaitFor && stillWaiting.compareAndSet(true, false)){
semaphore.release();
}
}
@Override
public void onFailure(Throwable t) {
thrown.add(t);
if (stillWaiting.compareAndSet(true,false)) {
semaphore.release();
}
}
}
private class MessageCountCallback implements FutureCallback<Integer> {
private int[] expectedCounts;
private int currentIteration;
public MessageCountCallback(int[] expectedCounts) {
this.expectedCounts = expectedCounts;
this.currentIteration = 0;
}
@Override
public void onSuccess(Integer result) {
if (currentIteration < expectedCounts.length){
if (result != expectedCounts[currentIteration]){
onFailure(new AssertionError("Wrong message count. Expected " + expectedCounts[currentIteration] + " but received " + result));
currentIteration = expectedCounts.length;
return;
}
}
currentIteration++;
if (currentIteration == expectedCounts.length)
semaphore.release();
}
@Override
public void onFailure(Throwable t) {
thrown.add(t);
semaphore.release();
}
}
@Test
public void testSync() throws SignatureException, CommunicationException, InterruptedException {
Timestamp timestamp = Timestamp.newBuilder()
.setSeconds(15252162)
.setNanos(85914)
.build();
BulletinBoardMessage msg = messageGenerator.generateRandomMessage(signers, timestamp, 10, 10);
MessageID msgID = localClient.postMessage(msg);
timestamp = Timestamp.newBuilder()
.setSeconds(51511653)
.setNanos(3625)
.build();
BulletinBoardMessage batchMessage = messageGenerator.generateRandomMessage(signers,timestamp, 100, 10);
MessageID batchMsgID = localClient.postAsBatch(batchMessage, 10);
BulletinBoardMessage test = localClient.readMessage(batchMsgID);
BulletinBoardMessage stub = localClient.readMessages(MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(batchMsgID.getID())
.build())
.build()).get(0);
BulletinBoardMessage test2 = localClient.readBatchData(stub);
synchronizer.subscribeToSyncStatus(new SyncStatusCallback(SyncStatus.SYNCHRONIZED));
int[] expectedCounts = {2,0};
synchronizer.subscribeToRemainingMessagesCount(new MessageCountCallback(expectedCounts));
Thread syncThread = new Thread(synchronizer);
syncThread.start();
if (!semaphore.tryAcquire(2, 4000, TimeUnit.MILLISECONDS)) {
thrown.add(new TimeoutException("Timeout occurred while waiting for synchronizer to sync."));
}
synchronizer.stop();
syncThread.join();
if (thrown.size() > 0) {
for (Throwable t : thrown)
System.err.println(t.getMessage());
assertThat("Exception thrown by Synchronizer: " + thrown.get(0).getMessage(), false);
}
List<BulletinBoardMessage> msgList = remoteClient.readMessages(MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(msgID.getID())
.build())
.build());
assertThat("Wrong number of messages returned.", msgList.size() == 1);
assertThat("Returned message is not equal to original one", messageComparator.compare(msgList.get(0),msg) == 0);
BulletinBoardMessage returnedBatchMsg = remoteClient.readMessage(batchMsgID);
assertThat("Returned batch does not equal original one.", messageComparator.compare(returnedBatchMsg, batchMessage) == 0);
}
@Test
public void testServerError() throws SignatureException, CommunicationException, InterruptedException {
Timestamp timestamp = Timestamp.newBuilder()
.setSeconds(945736256)
.setNanos(276788)
.build();
BulletinBoardMessage msg = messageGenerator.generateRandomMessage(signers, timestamp, 10, 10);
remoteClient.close();
synchronizer.subscribeToSyncStatus(new SyncStatusCallback(SyncStatus.SERVER_ERROR));
localClient.postMessage(msg);
Thread thread = new Thread(synchronizer);
thread.start();
if (!semaphore.tryAcquire(4000, TimeUnit.MILLISECONDS)) {
thrown.add(new TimeoutException("Timeout occurred while waiting for synchronizer to sync."));
}
synchronizer.stop();
thread.join();
}
@After
public void close() {
if (thrown.size() > 0) {
for (Throwable t : thrown) {
System.err.println(t.getMessage());
}
assertThat("Exception thrown by Synchronizer: " + thrown.get(0).getMessage(), false);
}
synchronizer.stop();
localClient.close();
remoteClient.close();
}
}

111
bulletin-board-client/src/test/java/meerkat/bulletinboard/CachedBulletinBoardClientTest.java Normal file
View File

@ -0,0 +1,111 @@
package meerkat.bulletinboard;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer;
import meerkat.bulletinboard.sqlserver.H2QueryProvider;
import meerkat.comm.CommunicationException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.security.SignatureException;
import java.util.LinkedList;
import java.util.List;
/**
* Created by Arbel on 6/27/2016.
*/
public class CachedBulletinBoardClientTest {
private static final int THREAD_NUM = 3;
private static final String LOCAL_DB_NAME = "localDB";
private static final String REMOTE_DB_NAME = "remoteDB";
private static final String QUEUE_DB_NAME = "queueDB";
private static final int SUBSRCIPTION_DELAY = 500;
private static final int SYNC_DELAY = 500;
// Testers
private CachedBulletinBoardClient cachedClient;
private GenericBulletinBoardClientTester clientTest;
private GenericSubscriptionClientTester subscriptionTester;
public CachedBulletinBoardClientTest() throws CommunicationException {
DeletableBulletinBoardServer localServer = new BulletinBoardSQLServer(new H2QueryProvider(LOCAL_DB_NAME));
localServer.init();
LocalBulletinBoardClient localClient = new LocalBulletinBoardClient(localServer, THREAD_NUM, SUBSRCIPTION_DELAY);
DeletableBulletinBoardServer remoteServer = new BulletinBoardSQLServer(new H2QueryProvider(REMOTE_DB_NAME));
remoteServer.init();
LocalBulletinBoardClient remoteClient = new LocalBulletinBoardClient(remoteServer, THREAD_NUM, SUBSRCIPTION_DELAY);
DeletableBulletinBoardServer queueServer = new BulletinBoardSQLServer(new H2QueryProvider(QUEUE_DB_NAME));
queueServer.init();
LocalBulletinBoardClient queueClient = new LocalBulletinBoardClient(queueServer, THREAD_NUM, SUBSRCIPTION_DELAY);
List<SubscriptionBulletinBoardClient> clientList = new LinkedList<>();
clientList.add(remoteClient);
BulletinBoardSubscriber subscriber = new ThreadedBulletinBoardSubscriber(clientList, localClient);
cachedClient = new CachedBulletinBoardClient(localClient, remoteClient, subscriber, queueClient, SYNC_DELAY, SYNC_DELAY);
subscriptionTester = new GenericSubscriptionClientTester(cachedClient);
clientTest = new GenericBulletinBoardClientTester(cachedClient, 87351);
}
// Test methods
/**
* Takes care of initializing the client and the test resources
*/
@Before
public void init(){
clientTest.init();
}
/**
* Closes the client and makes sure the test fails when an exception occurred in a separate thread
*/
@After
public void close() {
cachedClient.close();
clientTest.close();
}
@Test
public void testPost() {
clientTest.testPost();
}
@Test
public void testBatchPost() throws CommunicationException, SignatureException, InterruptedException {
clientTest.testBatchPost();
}
@Test
public void testCompleteBatchPost() throws CommunicationException, SignatureException, InterruptedException {
clientTest.testCompleteBatchPost();
}
@Test
public void testSubscription() throws SignatureException, CommunicationException {
// subscriptionTester.init();
// subscriptionTester.subscriptionTest();
// subscriptionTester.close();
}
}

497
bulletin-board-client/src/test/java/meerkat/bulletinboard/GenericBulletinBoardClientTester.java Normal file
View File

@ -0,0 +1,497 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.ByteString;
import com.google.protobuf.Timestamp;
import meerkat.comm.CommunicationException;
import meerkat.crypto.concrete.ECDSASignature;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto;
import meerkat.util.BulletinBoardMessageComparator;
import meerkat.util.BulletinBoardMessageGenerator;
import meerkat.util.BulletinBoardUtils;
import meerkat.bulletinboard.AsyncBulletinBoardClient.BatchIdentifier;
import java.io.IOException;
import java.io.InputStream;
import java.security.*;
import java.security.cert.CertificateException;
import java.util.*;
import java.util.concurrent.Semaphore;
import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.number.OrderingComparison.greaterThanOrEqualTo;
import static org.junit.Assert.*;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
*/
public class GenericBulletinBoardClientTester {
// Signature resources
private BulletinBoardSignature signers[];
private ByteString[] signerIDs;
private static String KEYFILE_EXAMPLE = "/certs/enduser-certs/user1-key-with-password-secret.p12";
private static String KEYFILE_EXAMPLE3 = "/certs/enduser-certs/user3-key-with-password-shh.p12";
private static String KEYFILE_PASSWORD1 = "secret";
private static String KEYFILE_PASSWORD3 = "shh";
private static String CERT1_PEM_EXAMPLE = "/certs/enduser-certs/user1.crt";
private static String CERT3_PEM_EXAMPLE = "/certs/enduser-certs/user3.crt";
// Client and callbacks
private AsyncBulletinBoardClient bulletinBoardClient;
private PostCallback postCallback;
private RedundancyCallback redundancyCallback;
private ReadCallback readCallback;
// Sync and misc
private Semaphore jobSemaphore;
private Vector<Throwable> thrown;
private Random random;
private BulletinBoardMessageGenerator generator;
private BulletinBoardDigest digest;
// Constructor
public GenericBulletinBoardClientTester(AsyncBulletinBoardClient bulletinBoardClient, int seed){
this.bulletinBoardClient = bulletinBoardClient;
signers = new GenericBulletinBoardSignature[2];
signerIDs = new ByteString[signers.length];
signers[0] = new GenericBulletinBoardSignature(new ECDSASignature());
signers[1] = new GenericBulletinBoardSignature(new ECDSASignature());
InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE);
char[] password = KEYFILE_PASSWORD1.toCharArray();
KeyStore.Builder keyStoreBuilder;
try {
keyStoreBuilder = signers[0].getPKCS12KeyStoreBuilder(keyStream, password);
signers[0].loadSigningCertificate(keyStoreBuilder);
signers[0].loadVerificationCertificates(getClass().getResourceAsStream(CERT1_PEM_EXAMPLE));
keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE3);
password = KEYFILE_PASSWORD3.toCharArray();
keyStoreBuilder = signers[1].getPKCS12KeyStoreBuilder(keyStream, password);
signers[1].loadSigningCertificate(keyStoreBuilder);
signers[1].loadVerificationCertificates(getClass().getResourceAsStream(CERT3_PEM_EXAMPLE));
for (int i = 0 ; i < signers.length ; i++) {
signerIDs[i] = signers[i].getSignerID();
}
} catch (IOException e) {
System.err.println("Failed reading from signature file " + e.getMessage());
fail("Failed reading from signature file " + e.getMessage());
} catch (CertificateException e) {
System.err.println("Failed reading certificate " + e.getMessage());
fail("Failed reading certificate " + e.getMessage());
} catch (KeyStoreException e) {
System.err.println("Failed reading keystore " + e.getMessage());
fail("Failed reading keystore " + e.getMessage());
} catch (NoSuchAlgorithmException e) {
System.err.println("Couldn't find signing algorithm " + e.getMessage());
fail("Couldn't find signing algorithm " + e.getMessage());
} catch (UnrecoverableKeyException e) {
System.err.println("Couldn't find signing key " + e.getMessage());
fail("Couldn't find signing key " + e.getMessage());
}
this.random = new Random(seed);
this.generator = new BulletinBoardMessageGenerator(random);
this.digest = new GenericBulletinBoardDigest(new SHA256Digest());
}
// Callback definitions
protected void genericHandleFailure(Throwable t){
System.err.println(t.getCause() + " " + t.getMessage());
thrown.add(t);
jobSemaphore.release();
}
private class PostCallback implements FutureCallback<Boolean>{
private boolean isAssert;
private boolean assertValue;
public PostCallback() {
this(false);
}
public PostCallback(boolean isAssert) {
this(isAssert,true);
}
public PostCallback(boolean isAssert, boolean assertValue) {
this.isAssert = isAssert;
this.assertValue = assertValue;
}
@Override
public void onSuccess(Boolean msg) {
System.err.println("Post operation completed");
if (isAssert) {
if (assertValue && !msg) {
genericHandleFailure(new AssertionError("Post operation failed"));
} else if (!assertValue && msg){
genericHandleFailure(new AssertionError("Post operation succeeded unexpectedly"));
} else {
jobSemaphore.release();
}
} else {
jobSemaphore.release();
}
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
}
private class RedundancyCallback implements FutureCallback<Float>{
private float minRedundancy;
public RedundancyCallback(float minRedundancy) {
this.minRedundancy = minRedundancy;
}
@Override
public void onSuccess(Float redundancy) {
System.err.println("Redundancy found is: " + redundancy);
jobSemaphore.release();
assertThat(redundancy, greaterThanOrEqualTo(minRedundancy));
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
}
private class ReadCallback implements FutureCallback<List<BulletinBoardMessage>>{
private List<BulletinBoardMessage> expectedMsgList;
public ReadCallback(List<BulletinBoardMessage> expectedMsgList) {
this.expectedMsgList = expectedMsgList;
}
@Override
public void onSuccess(List<BulletinBoardMessage> messages) {
System.err.println(messages);
jobSemaphore.release();
BulletinBoardMessageComparator msgComparator = new BulletinBoardMessageComparator();
assertThat(messages.size(), is(expectedMsgList.size()));
Iterator<BulletinBoardMessage> expectedMessageIterator = expectedMsgList.iterator();
Iterator<BulletinBoardMessage> receivedMessageIterator = messages.iterator();
while (expectedMessageIterator.hasNext()) {
assertThat(msgComparator.compare(expectedMessageIterator.next(), receivedMessageIterator.next()), is(0));
}
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
}
private class ReadBatchCallback implements FutureCallback<BulletinBoardMessage>{
private BulletinBoardMessage expectedMsg;
public ReadBatchCallback(BulletinBoardMessage expectedMsg) {
this.expectedMsg = expectedMsg;
}
@Override
public void onSuccess(BulletinBoardMessage msg) {
BulletinBoardMessageComparator msgComparator = new BulletinBoardMessageComparator();
if (msgComparator.compare(msg, expectedMsg) != 0) {
genericHandleFailure(new AssertionError("Batch read returned different message.\nExpected:" + expectedMsg + "\nRecieved:" + msg + "\n"));
} else {
jobSemaphore.release();
}
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
}
// Test methods
/**
* Takes care of initializing the client and the test resources
*/
public void init(){
random = new Random(0); // We use insecure randomness in tests for repeatability
postCallback = new PostCallback();
redundancyCallback = new RedundancyCallback((float) 1.0);
thrown = new Vector<>();
jobSemaphore = new Semaphore(0);
}
/**
* Closes the client and makes sure the test fails when an exception occurred in a separate thread
*/
public void close() {
if (thrown.size() > 0) {
for (Throwable t : thrown){
System.err.println(t.getMessage());
}
assert false;
}
}
/**
* Tests the standard post, redundancy and read methods
*/
public void testPost() {
byte[] b1 = {(byte) 1, (byte) 2, (byte) 3, (byte) 4};
byte[] b2 = {(byte) 11, (byte) 12, (byte) 13, (byte) 14};
byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24};
BulletinBoardMessage msg;
MessageFilterList filterList;
List<BulletinBoardMessage> msgList;
MessageID messageID;
msg = BulletinBoardMessage.newBuilder()
.setMsg(UnsignedBulletinBoardMessage.newBuilder()
.addTag("Signature")
.addTag("Trustee")
.setData(ByteString.copyFrom(b1))
.setTimestamp(Timestamp.newBuilder()
.setSeconds(20)
.setNanos(30)
.build())
.build())
.addSig(Crypto.Signature.newBuilder()
.setType(Crypto.SignatureType.DSA)
.setData(ByteString.copyFrom(b2))
.setSignerId(ByteString.copyFrom(b3))
.build())
.addSig(Crypto.Signature.newBuilder()
.setType(Crypto.SignatureType.ECDSA)
.setData(ByteString.copyFrom(b3))
.setSignerId(ByteString.copyFrom(b2))
.build())
.build();
messageID = bulletinBoardClient.postMessage(msg,postCallback);
try {
jobSemaphore.acquire();
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());
}
bulletinBoardClient.getRedundancy(messageID,redundancyCallback);
filterList = MessageFilterList.newBuilder()
.addFilter(
MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag("Signature")
.build()
)
.addFilter(
MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag("Trustee")
.build()
)
.build();
msgList = new LinkedList<>();
msgList.add(msg);
readCallback = new ReadCallback(msgList);
bulletinBoardClient.readMessages(filterList, readCallback);
try {
jobSemaphore.acquire(2);
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());
}
}
/**
* Tests posting a batch by parts
* @throws CommunicationException, SignatureException, InterruptedException
*/
public void testBatchPost() throws CommunicationException, SignatureException, InterruptedException {
final int BATCH_LENGTH = 100;
final int CHUNK_SIZE = 10;
final int TAG_NUM = 10;
final Timestamp timestamp = Timestamp.newBuilder()
.setSeconds(141515)
.setNanos(859018)
.build();
final BulletinBoardMessage msg = generator.generateRandomMessage(signers, timestamp, BATCH_LENGTH, TAG_NUM);
// Begin batch
bulletinBoardClient.beginBatch(msg.getMsg().getTagList(), new FutureCallback<BatchIdentifier>() {
@Override
public void onSuccess(final BatchIdentifier identifier) {
bulletinBoardClient.postBatchData(identifier, BulletinBoardUtils.breakToBatch(msg, CHUNK_SIZE), new FutureCallback<Boolean>() {
@Override
public void onSuccess(Boolean result) {
bulletinBoardClient.closeBatch(identifier, msg.getMsg().getTimestamp(), msg.getSigList(), new FutureCallback<Boolean>() {
@Override
public void onSuccess(Boolean result) {
jobSemaphore.release();
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
});
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
});
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
});
jobSemaphore.acquire();
digest.reset();
digest.update(msg);
bulletinBoardClient.readMessage(digest.digestAsMessageID(), new ReadBatchCallback(msg));
jobSemaphore.acquire();
}
/**
* Posts a complete batch message
* Checks reading of the message in two parts
* @throws CommunicationException, SignatureException, InterruptedException
*/
public void testCompleteBatchPost() throws CommunicationException, SignatureException, InterruptedException {
final int BATCH_LENGTH = 100;
final int CHUNK_SIZE = 99;
final int TAG_NUM = 8;
final Timestamp timestamp = Timestamp.newBuilder()
.setSeconds(7776151)
.setNanos(252616)
.build();
final BulletinBoardMessage msg = generator.generateRandomMessage(signers, timestamp, BATCH_LENGTH, TAG_NUM);
// Post batch
MessageID msgID = bulletinBoardClient.postAsBatch(msg, CHUNK_SIZE, postCallback);
jobSemaphore.acquire();
// Read batch
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(msgID.getID())
.build())
.build();
bulletinBoardClient.readMessages(filterList, new FutureCallback<List<BulletinBoardMessage>>() {
@Override
public void onSuccess(List<BulletinBoardMessage> msgList) {
if (msgList.size() != 1) {
genericHandleFailure(new AssertionError("Wrong number of stubs returned. Expected: 1; Found: " + msgList.size()));
} else {
BulletinBoardMessage retrievedMsg = msgList.get(0);
bulletinBoardClient.readBatchData(retrievedMsg, new ReadBatchCallback(msg));
}
}
@Override
public void onFailure(Throwable t) {
genericHandleFailure(t);
}
});
jobSemaphore.acquire();
}
}

226
bulletin-board-client/src/test/java/meerkat/bulletinboard/GenericSubscriptionClientTester.java Normal file
View File

@ -0,0 +1,226 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.FutureCallback;
import com.google.protobuf.ByteString;
import com.google.protobuf.Timestamp;
import meerkat.comm.CommunicationException;
import meerkat.crypto.concrete.ECDSASignature;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.util.BulletinBoardMessageComparator;
import meerkat.util.BulletinBoardMessageGenerator;
import java.io.IOException;
import java.io.InputStream;
import java.security.*;
import java.security.cert.CertificateException;
import java.util.*;
import java.util.concurrent.Semaphore;
import static org.junit.Assert.fail;
/**
* Created by Arbel Deutsch Peled on 22-Mar-16.
*/
public class GenericSubscriptionClientTester {
private BulletinBoardSignature signers[];
private ByteString[] signerIDs;
private static String KEYFILE_EXAMPLE = "/certs/enduser-certs/user1-key-with-password-secret.p12";
private static String KEYFILE_EXAMPLE3 = "/certs/enduser-certs/user3-key-with-password-shh.p12";
private static String KEYFILE_PASSWORD1 = "secret";
private static String KEYFILE_PASSWORD3 = "shh";
private static String CERT1_PEM_EXAMPLE = "/certs/enduser-certs/user1.crt";
private static String CERT3_PEM_EXAMPLE = "/certs/enduser-certs/user3.crt";
private SubscriptionBulletinBoardClient bulletinBoardClient;
private Random random;
private BulletinBoardMessageGenerator generator;
private Semaphore jobSemaphore;
private Vector<Throwable> thrown;
public GenericSubscriptionClientTester(SubscriptionBulletinBoardClient bulletinBoardClient){
this.bulletinBoardClient = bulletinBoardClient;
signers = new BulletinBoardSignature[2];
signerIDs = new ByteString[signers.length];
signers[0] = new GenericBulletinBoardSignature(new ECDSASignature());
signers[1] = new GenericBulletinBoardSignature(new ECDSASignature());
InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE);
char[] password = KEYFILE_PASSWORD1.toCharArray();
KeyStore.Builder keyStoreBuilder;
try {
keyStoreBuilder = signers[0].getPKCS12KeyStoreBuilder(keyStream, password);
signers[0].loadSigningCertificate(keyStoreBuilder);
signers[0].loadVerificationCertificates(getClass().getResourceAsStream(CERT1_PEM_EXAMPLE));
keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE3);
password = KEYFILE_PASSWORD3.toCharArray();
keyStoreBuilder = signers[1].getPKCS12KeyStoreBuilder(keyStream, password);
signers[1].loadSigningCertificate(keyStoreBuilder);
signers[1].loadVerificationCertificates(getClass().getResourceAsStream(CERT3_PEM_EXAMPLE));
for (int i = 0 ; i < signers.length ; i++) {
signerIDs[i] = signers[i].getSignerID();
}
} catch (IOException e) {
System.err.println("Failed reading from signature file " + e.getMessage());
fail("Failed reading from signature file " + e.getMessage());
} catch (CertificateException e) {
System.err.println("Failed reading certificate " + e.getMessage());
fail("Failed reading certificate " + e.getMessage());
} catch (KeyStoreException e) {
System.err.println("Failed reading keystore " + e.getMessage());
fail("Failed reading keystore " + e.getMessage());
} catch (NoSuchAlgorithmException e) {
System.err.println("Couldn't find signing algorithm " + e.getMessage());
fail("Couldn't find signing algorithm " + e.getMessage());
} catch (UnrecoverableKeyException e) {
System.err.println("Couldn't find signing key " + e.getMessage());
fail("Couldn't find signing key " + e.getMessage());
}
}
/**
* Takes care of initializing the client and the test resources
*/
public void init(){
random = new Random(0); // We use insecure randomness in tests for repeatability
generator = new BulletinBoardMessageGenerator(random);
thrown = new Vector<>();
jobSemaphore = new Semaphore(0);
}
/**
* Closes the client and makes sure the test fails when an exception occurred in a separate thread
*/
public void close() {
if (thrown.size() > 0) {
assert false;
}
}
private class SubscriptionCallback implements FutureCallback<List<BulletinBoardMessage>>{
private int stage;
private final List<List<BulletinBoardMessage>> expectedMessages;
private final List<BulletinBoardMessage> messagesToPost;
private final BulletinBoardMessageComparator comparator;
public SubscriptionCallback(List<List<BulletinBoardMessage>> expectedMessages, List<BulletinBoardMessage> messagesToPost) {
this.expectedMessages = expectedMessages;
this.messagesToPost = messagesToPost;
this.stage = 0;
this.comparator = new BulletinBoardMessageComparator();
}
@Override
public void onSuccess(List<BulletinBoardMessage> result) {
if (stage >= expectedMessages.size())
return;
// Check for consistency
List<BulletinBoardMessage> expectedMsgList = expectedMessages.get(stage);
if (expectedMsgList.size() != result.size()){
onFailure(new AssertionError("Received wrong number of messages"));
return;
}
Iterator<BulletinBoardMessage> expectedMessageIterator = expectedMsgList.iterator();
Iterator<BulletinBoardMessage> receivedMessageIterator = result.iterator();
while (expectedMessageIterator.hasNext()) {
if(comparator.compare(expectedMessageIterator.next(), receivedMessageIterator.next()) != 0){
onFailure(new AssertionError("Received unexpected message"));
return;
}
}
// Post new message
try {
if (stage < messagesToPost.size()) {
bulletinBoardClient.postMessage(messagesToPost.get(stage));
}
} catch (CommunicationException e) {
onFailure(e);
return;
}
stage++;
jobSemaphore.release();
}
@Override
public void onFailure(Throwable t) {
System.err.println(t.getCause() + " " + t.getMessage());
thrown.add(t);
jobSemaphore.release();
stage = expectedMessages.size();
}
}
public void subscriptionTest() throws SignatureException, CommunicationException {
final String COMMON_TAG = "SUBSCRIPTION_TEST";
List<String> tags = new LinkedList<>();
tags.add(COMMON_TAG);
BulletinBoardMessage msg1 = generator.generateRandomMessage(signers, Timestamp.newBuilder().setSeconds(1000).setNanos(900).build(), 10, 4, tags);
BulletinBoardMessage msg2 = generator.generateRandomMessage(signers, Timestamp.newBuilder().setSeconds(800).setNanos(300).build(), 10, 4);
BulletinBoardMessage msg3 = generator.generateRandomMessage(signers, Timestamp.newBuilder().setSeconds(2000).setNanos(0).build(), 10, 4, tags);
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag(COMMON_TAG)
.build())
.build();
List<List<BulletinBoardMessage>> expectedMessages = new ArrayList<>(3);
expectedMessages.add(new LinkedList<>());
expectedMessages.add(new LinkedList<>());
expectedMessages.add(new LinkedList<>());
expectedMessages.get(0).add(msg1);
expectedMessages.get(2).add(msg3);
List<BulletinBoardMessage> messagesToPost = new ArrayList<>(2);
messagesToPost.add(msg2);
messagesToPost.add(msg3);
bulletinBoardClient.postMessage(msg1);
bulletinBoardClient.subscribe(filterList, new SubscriptionCallback(expectedMessages, messagesToPost));
try {
jobSemaphore.acquire(3);
} catch (InterruptedException e) {
System.err.println(e.getCause() + " " + e.getMessage());
}
}
}

90
bulletin-board-client/src/test/java/meerkat/bulletinboard/LocalBulletinBoardClientTest.java Normal file
View File

@ -0,0 +1,90 @@
package meerkat.bulletinboard;
import meerkat.bulletinboard.sqlserver.*;
import meerkat.comm.CommunicationException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.security.SignatureException;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
*/
public class LocalBulletinBoardClientTest {
private static final int THREAD_NUM = 3;
private static final String DB_NAME = "TestDB";
private static final int SUBSRCIPTION_DELAY = 3000;
// Testers
private GenericBulletinBoardClientTester clientTest;
private GenericSubscriptionClientTester subscriptionTester;
public LocalBulletinBoardClientTest() throws CommunicationException {
H2QueryProvider queryProvider = new H2QueryProvider(DB_NAME);
DeletableBulletinBoardServer server = new BulletinBoardSQLServer(queryProvider);
server.init();
LocalBulletinBoardClient client = new LocalBulletinBoardClient(server, THREAD_NUM, SUBSRCIPTION_DELAY);
subscriptionTester = new GenericSubscriptionClientTester(client);
clientTest = new GenericBulletinBoardClientTester(client, 98354);
}
// Test methods
/**
* Takes care of initializing the client and the test resources
*/
@Before
public void init(){
clientTest.init();
}
/**
* Closes the client and makes sure the test fails when an exception occurred in a separate thread
*/
@After
public void close() {
clientTest.close();
}
@Test
public void testPost() {
clientTest.testPost();
}
@Test
public void testBatchPost() throws CommunicationException, SignatureException, InterruptedException {
clientTest.testBatchPost();
}
@Test
public void testCompleteBatchPost() throws CommunicationException, SignatureException, InterruptedException {
clientTest.testCompleteBatchPost();
}
@Test
public void testSubscription() throws SignatureException, CommunicationException {
subscriptionTester.init();
subscriptionTester.subscriptionTest();
subscriptionTester.close();
}
}

104
bulletin-board-client/src/test/java/meerkat/bulletinboard/SingleServerBulletinBoardClientIntegrationTest.java Normal file
View File

@ -0,0 +1,104 @@
package meerkat.bulletinboard;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.Voting.BulletinBoardClientParams;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.security.SignatureException;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Executors;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
*/
public class SingleServerBulletinBoardClientIntegrationTest {
// Server data
private static final String PROP_GETTY_URL = "gretty.httpBaseURI";
private static final String DEFAULT_BASE_URL = "http://localhost:8081";
private static final String BASE_URL = System.getProperty(PROP_GETTY_URL, DEFAULT_BASE_URL);
private static final int THREAD_NUM = 3;
private static final long FAIL_DELAY = 3000;
private static final long SUBSCRIPTION_INTERVAL = 500;
// Testers
private GenericBulletinBoardClientTester clientTest;
private GenericSubscriptionClientTester subscriptionTester;
public SingleServerBulletinBoardClientIntegrationTest(){
SingleServerBulletinBoardClient client = new SingleServerBulletinBoardClient(THREAD_NUM, FAIL_DELAY, SUBSCRIPTION_INTERVAL);
List<String> testDB = new LinkedList<>();
testDB.add(BASE_URL);
client.init(BulletinBoardClientParams.newBuilder()
.addAllBulletinBoardAddress(testDB)
.setMinRedundancy((float) 1.0)
.build());
clientTest = new GenericBulletinBoardClientTester(client, 981541);
subscriptionTester = new GenericSubscriptionClientTester(client);
}
// Test methods
/**
* Takes care of initializing the client and the test resources
*/
@Before
public void init(){
clientTest.init();
}
/**
* Closes the client and makes sure the test fails when an exception occurred in a separate thread
*/
@After
public void close() {
clientTest.close();
}
@Test
public void testPost() {
clientTest.testPost();
}
@Test
public void testBatchPost() throws CommunicationException, SignatureException, InterruptedException {
clientTest.testBatchPost();
}
@Test
public void testCompleteBatchPost() throws CommunicationException, SignatureException, InterruptedException {
clientTest.testCompleteBatchPost();
}
@Test
public void testSubscription() throws SignatureException, CommunicationException {
subscriptionTester.init();
subscriptionTester.subscriptionTest();
subscriptionTester.close();
}
}

89
bulletin-board-client/src/test/java/meerkat/bulletinboard/ThreadedBulletinBoardClientIntegrationTest.java Normal file
View File

@ -0,0 +1,89 @@
package meerkat.bulletinboard;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.Voting.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.security.SignatureException;
import java.util.LinkedList;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 05-Dec-15.
*/
public class ThreadedBulletinBoardClientIntegrationTest {
// Server data
private static String PROP_GETTY_URL = "gretty.httpBaseURI";
private static String DEFAULT_BASE_URL = "http://localhost:8081";
private static String BASE_URL = System.getProperty(PROP_GETTY_URL, DEFAULT_BASE_URL);
// Tester
private GenericBulletinBoardClientTester clientTest;
public ThreadedBulletinBoardClientIntegrationTest(){
ThreadedBulletinBoardClient client = new ThreadedBulletinBoardClient(3,0,500);
List<String> testDB = new LinkedList<>();
testDB.add(BASE_URL);
client.init(BulletinBoardClientParams.newBuilder()
.addAllBulletinBoardAddress(testDB)
.setMinRedundancy((float) 1.0)
.build());
clientTest = new GenericBulletinBoardClientTester(client, 52351);
}
// Test methods
/**
* Takes care of initializing the client and the test resources
*/
@Before
public void init(){
clientTest.init();
}
/**
* Closes the client and makes sure the test fails when an exception occurred in a separate thread
*/
@After
public void close() {
clientTest.close();
}
@Test
public void testPost() {
clientTest.testPost();
}
@Test
public void testBatchPost() throws CommunicationException, SignatureException, InterruptedException {
clientTest.testBatchPost();
}
@Test
public void testCompleteBatchPost() throws CommunicationException, SignatureException, InterruptedException {
clientTest.testCompleteBatchPost();
}
}

28
bulletin-board-server/build.gradle
View File

@ -48,9 +48,10 @@ dependencies {
// JDBC connections
compile 'org.springframework:spring-jdbc:4.2.+'
compile 'org.xerial:sqlite-jdbc:3.7.+'
compile 'org.xerial:sqlite-jdbc:3.8.+'
compile 'mysql:mysql-connector-java:5.1.+'
compile 'com.h2database:h2:1.0.+'
compile 'org.apache.commons:commons-dbcp2:2.0.+'
// Servlets
compile 'javax.servlet:javax.servlet-api:3.0.+'
@ -79,9 +80,30 @@ test {
exclude '**/*IntegrationTest*'
}
task myTest(type: Test) {
include '**/*MySQL*Test*'
outputs.upToDateWhen { false }
}
task h2Test(type: Test) {
include '**/*H2*Test*'
outputs.upToDateWhen { false }
}
task liteTest(type: Test) {
include '**/*SQLite*Test*'
outputs.upToDateWhen { false }
}
task dbTest(type: Test) {
include '**/*H2*Test*'
include '**/*MySql*Test'
include '**/*MySQL*Test*'
include '**/*SQLite*Test*'
outputs.upToDateWhen { false }
}
task manualIntegration(type: Test) {
include '**/*IntegrationTest*'
}
task integrationTest(type: Test) {
@ -237,5 +259,3 @@ publishing {
}
}

886
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/BulletinBoardSQLServer.java
View File

File diff suppressed because it is too large Load Diff

179
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/H2QueryProvider.java
View File

@ -1,12 +1,10 @@
package meerkat.bulletinboard.sqlserver;
import meerkat.protobuf.BulletinBoardAPI.FilterType;
import org.h2.jdbcx.JdbcDataSource;
import javax.naming.Context;
import javax.naming.InitialContext;
import org.apache.commons.dbcp2.BasicDataSource;
import javax.naming.NamingException;
import javax.sql.DataSource;
import java.text.MessageFormat;
import java.util.LinkedList;
import java.util.List;
@ -28,32 +26,123 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
switch(queryType) {
case ADD_SIGNATURE:
return "INSERT INTO SignatureTable (EntryNum, SignerId, Signature)"
+ " SELECT DISTINCT :EntryNum AS Entry, :SignerId AS Id, :Signature AS Sig FROM UtilityTable AS Temp"
+ " WHERE NOT EXISTS"
+ " (SELECT 1 FROM SignatureTable AS SubTable WHERE SubTable.SignerId = :SignerId AND SubTable.EntryNum = :EntryNum)";
return MessageFormat.format(
"INSERT INTO SignatureTable (EntryNum, SignerId, Signature)"
+ " SELECT DISTINCT :{0} AS Entry, :{1} AS Id, :{2} AS Sig FROM UtilityTable AS Temp"
+ " WHERE NOT EXISTS"
+ " (SELECT 1 FROM SignatureTable AS SubTable WHERE SubTable.EntryNum = :{0} AND SubTable.SignerId = :{1})",
QueryType.ADD_SIGNATURE.getParamName(0),
QueryType.ADD_SIGNATURE.getParamName(1),
QueryType.ADD_SIGNATURE.getParamName(2));
case CONNECT_TAG:
return "INSERT INTO MsgTagTable (TagId, EntryNum)"
+ " SELECT DISTINCT TagTable.TagId, :EntryNum AS NewEntry FROM TagTable WHERE Tag = :Tag"
+ " AND NOT EXISTS (SELECT 1 FROM MsgTagTable AS SubTable WHERE SubTable.TagId = TagTable.TagId"
+ " AND SubTable.EntryNum = :EntryNum)";
return MessageFormat.format(
"INSERT INTO MsgTagTable (TagId, EntryNum)"
+ " SELECT DISTINCT TagTable.TagId, :{0} AS NewEntry FROM TagTable WHERE Tag = :{1}"
+ " AND NOT EXISTS (SELECT 1 FROM MsgTagTable AS SubTable WHERE SubTable.TagId = TagTable.TagId"
+ " AND SubTable.EntryNum = :{0})",
QueryType.CONNECT_TAG.getParamName(0),
QueryType.CONNECT_TAG.getParamName(1));
case FIND_MSG_ID:
return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId";
return MessageFormat.format(
"SELECT EntryNum From MsgTable WHERE MsgId = :{0}",
QueryType.FIND_MSG_ID.getParamName(0));
case FIND_TAG_ID:
return MessageFormat.format(
"SELECT TagId FROM TagTable WHERE Tag = :{0}",
QueryType.FIND_TAG_ID.getParamName(0));
case GET_MESSAGES:
return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable";
case COUNT_MESSAGES:
return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable";
case GET_MESSAGE_STUBS:
return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable";
case GET_SIGNATURES:
return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum";
return MessageFormat.format(
"SELECT Signature FROM SignatureTable WHERE EntryNum = :{0}",
QueryType.GET_SIGNATURES.getParamName(0));
case INSERT_MSG:
return "INSERT INTO MsgTable (MsgId, Msg) VALUES(:MsgId,:Msg)";
return MessageFormat.format(
"INSERT INTO MsgTable (MsgId, ExactTime, Msg) VALUES(:{0}, :{1}, :{2})",
QueryType.INSERT_MSG.getParamName(0),
QueryType.INSERT_MSG.getParamName(1),
QueryType.INSERT_MSG.getParamName(2));
case DELETE_MSG_BY_ENTRY:
return MessageFormat.format(
"DELETE FROM MsgTable WHERE EntryNum = :{0}",
QueryType.DELETE_MSG_BY_ENTRY.getParamName(0));
case DELETE_MSG_BY_ID:
return MessageFormat.format(
"DELETE FROM MsgTable WHERE MsgId = :{0}",
QueryType.DELETE_MSG_BY_ID.getParamName(0));
case INSERT_NEW_TAG:
return "INSERT INTO TagTable(Tag) SELECT DISTINCT :Tag AS NewTag FROM UtilityTable WHERE"
+ " NOT EXISTS (SELECT 1 FROM TagTable AS SubTable WHERE SubTable.Tag = :Tag)";
return MessageFormat.format(
"INSERT INTO TagTable(Tag) SELECT DISTINCT :Tag AS NewTag FROM UtilityTable WHERE"
+ " NOT EXISTS (SELECT 1 FROM TagTable AS SubTable WHERE SubTable.Tag = :{0})",
QueryType.INSERT_NEW_TAG.getParamName(0));
case GET_LAST_MESSAGE_ENTRY:
return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable";
case GET_BATCH_MESSAGE_DATA_BY_MSG_ID:
return MessageFormat.format(
"SELECT Data FROM BatchTable"
+ " INNER JOIN MsgTable ON MsgTable.EntryNum = BatchTable.EntryNum"
+ " WHERE MsgTable.MsgId = :{0} AND BatchTable.SerialNum >= :{1}"
+ " ORDER BY BatchTable.SerialNum ASC",
QueryType.GET_BATCH_MESSAGE_DATA_BY_MSG_ID.getParamName(0),
QueryType.GET_BATCH_MESSAGE_DATA_BY_MSG_ID.getParamName(1));
case GET_BATCH_MESSAGE_DATA_BY_BATCH_ID:
return MessageFormat.format(
"SELECT Data FROM BatchTable"
+ " WHERE BatchId = :{0} AND SerialNum >= :{1}"
+ " ORDER BY BatchTable.SerialNum ASC",
QueryType.GET_BATCH_MESSAGE_DATA_BY_BATCH_ID.getParamName(0),
QueryType.GET_BATCH_MESSAGE_DATA_BY_BATCH_ID.getParamName(1));
case INSERT_BATCH_DATA:
return MessageFormat.format(
"INSERT INTO BatchTable (BatchId, SerialNum, Data) VALUES (:{0}, :{1}, :{2})",
QueryType.INSERT_BATCH_DATA.getParamName(0),
QueryType.INSERT_BATCH_DATA.getParamName(1),
QueryType.INSERT_BATCH_DATA.getParamName(2));
case CHECK_BATCH_LENGTH:
return MessageFormat.format(
"SELECT COUNT(Data) AS BatchLength FROM BatchTable WHERE BatchId = :{0}",
QueryType.CHECK_BATCH_LENGTH.getParamName(0));
case CHECK_BATCH_OPEN:
return MessageFormat.format(
"SELECT COUNT(BatchId) AS batchCount FROM BatchTagTable WHERE BatchId = :{0}",
QueryType.CHECK_BATCH_OPEN.getParamName(0));
case STORE_BATCH_TAGS:
return MessageFormat.format(
"INSERT INTO BatchTagTable (Tags) VALUES (:{0})",
QueryType.STORE_BATCH_TAGS.getParamName(0));
case GET_BATCH_TAGS:
return MessageFormat.format(
"SELECT Tags FROM BatchTagTable WHERE BatchId = :{0}",
QueryType.GET_BATCH_TAGS.getParamName(0));
case ADD_ENTRY_NUM_TO_BATCH:
return MessageFormat.format(
"UPDATE BatchTable SET EntryNum = :{1} WHERE BatchId = :{0}",
QueryType.ADD_ENTRY_NUM_TO_BATCH.getParamName(0),
QueryType.ADD_ENTRY_NUM_TO_BATCH.getParamName(1));
default:
throw new IllegalArgumentException("Cannot serve a query of type " + queryType);
@ -71,10 +160,12 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
return "MsgTable.EntryNum = :EntryNum" + serialString;
case MAX_ENTRY:
return "MsgTable.EntryNum <= :EntryNum" + serialString;
case MIN_ENTRY:
return "MsgTable.EntryNum >= :EntryNum" + serialString;
case MAX_MESSAGES:
return "LIMIT :Limit" + serialString;
case MSG_ID:
return "MsgTable.MsgId = MsgId" + serialString;
return "MsgTable.MsgId = :MsgId" + serialString;
case SIGNER_ID:
return "EXISTS (SELECT 1 FROM SignatureTable"
+ " WHERE SignatureTable.SignerId = :SignerId" + serialString + " AND SignatureTable.EntryNum = MsgTable.EntryNum)";
@ -82,6 +173,13 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
return "EXISTS (SELECT 1 FROM TagTable"
+ " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId"
+ " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)";
case BEFORE_TIME:
return "MsgTable.ExactTime <= :TimeStamp" + serialString;
case AFTER_TIME:
return "MsgTable.ExactTime >= :TimeStamp" + serialString;
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
@ -94,6 +192,7 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
switch(filterType) {
case EXACT_ENTRY: // Go through
case MAX_ENTRY: // Go through
case MIN_ENTRY: // Go through
case MAX_MESSAGES:
return "INT";
@ -104,6 +203,11 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
case TAG:
return "VARCHAR";
case AFTER_TIME: // Go through
case BEFORE_TIME:
return "TIMESTAMP";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
@ -113,10 +217,13 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
@Override
public DataSource getDataSource() {
JdbcDataSource dataSource = new JdbcDataSource();
dataSource.setURL("jdbc:h2:~/" + dbName);
BasicDataSource dataSource = new BasicDataSource();
dataSource.setDriverClassName("org.h2.Driver");
dataSource.setUrl("jdbc:h2:mem:" + dbName);
return dataSource;
}
@ -124,20 +231,30 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
public List<String> getSchemaCreationCommands() {
List<String> list = new LinkedList<String>();
list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB UNIQUE, Msg BLOB)");
list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY,"
+ " MsgId TINYBLOB UNIQUE, ExactTime TIMESTAMP, Msg BLOB)");
list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INT NOT NULL AUTO_INCREMENT PRIMARY KEY, Tag VARCHAR(50) UNIQUE)");
list.add("CREATE TABLE IF NOT EXISTS MsgTagTable (EntryNum INT, TagId INT,"
+ " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum),"
+ " FOREIGN KEY (TagId) REFERENCES TagTable(TagId),"
+ " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum) ON DELETE CASCADE,"
+ " FOREIGN KEY (TagId) REFERENCES TagTable(TagId) ON DELETE CASCADE,"
+ " UNIQUE (EntryNum, TagID))");
list.add("CREATE TABLE IF NOT EXISTS SignatureTable (EntryNum INT, SignerId TINYBLOB, Signature TINYBLOB UNIQUE,"
+ " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))");
+ " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum) ON DELETE CASCADE)");
list.add("CREATE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId)");
list.add("CREATE UNIQUE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId, EntryNum)");
list.add("CREATE UNIQUE INDEX IF NOT EXISTS SignatureIndex ON SignatureTable(SignerId, EntryNum)");
list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (BatchId INT AUTO_INCREMENT PRIMARY KEY, Tags BLOB)");
list.add("CREATE TABLE IF NOT EXISTS BatchTable (BatchId INT, EntryNum INT, SerialNum INT, Data BLOB,"
+ " UNIQUE(BatchId, SerialNum),"
+ " FOREIGN KEY (BatchId) REFERENCES BatchTagTable(BatchId) ON DELETE CASCADE)");
list.add("CREATE INDEX IF NOT EXISTS BatchDataIndex ON BatchTable(EntryNum, SerialNum)");
// This is used to create a simple table with one entry.
// It is used for implementing a workaround for the missing INSERT IGNORE syntax
@ -152,10 +269,20 @@ public class H2QueryProvider implements BulletinBoardSQLServer.SQLQueryProvider
List<String> list = new LinkedList<String>();
list.add("DROP TABLE IF EXISTS UtilityTable");
list.add("DROP INDEX IF EXISTS SignerIdIndex");
list.add("DROP INDEX IF EXISTS BatchDataIndex");
list.add("DROP TABLE IF EXISTS BatchTable");
list.add("DROP INDEX IF EXISTS BatchTagIndex");
list.add("DROP TABLE IF EXISTS BatchTagTable");
list.add("DROP TABLE IF EXISTS MsgTagTable");
list.add("DROP INDEX IF EXISTS SignerIdIndex");
list.add("DROP TABLE IF EXISTS SignatureTable");
list.add("DROP TABLE IF EXISTS TagTable");
list.add("DROP TABLE IF EXISTS MsgTable");
return list;

164
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/MySQLQueryProvider.java
View File

@ -1,10 +1,11 @@
package meerkat.bulletinboard.sqlserver;
import com.mysql.jdbc.jdbc2.optional.MysqlDataSource;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider;
import meerkat.protobuf.BulletinBoardAPI.FilterType;
import org.apache.commons.dbcp2.BasicDataSource;
import javax.sql.DataSource;
import java.text.MessageFormat;
import java.util.LinkedList;
import java.util.List;
@ -32,21 +33,120 @@ public class MySQLQueryProvider implements SQLQueryProvider {
public String getSQLString(QueryType queryType) throws IllegalArgumentException{
switch(queryType) {
case ADD_SIGNATURE:
return "INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:EntryNum, :SignerId, :Signature)";
return MessageFormat.format(
"INSERT IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:{0}, :{1}, :{2})",
QueryType.ADD_SIGNATURE.getParamName(0),
QueryType.ADD_SIGNATURE.getParamName(1),
QueryType.ADD_SIGNATURE.getParamName(2));
case CONNECT_TAG:
return "INSERT IGNORE INTO MsgTagTable (TagId, EntryNum)"
+ " SELECT TagTable.TagId, :EntryNum AS EntryNum FROM TagTable WHERE Tag = :Tag";
return MessageFormat.format(
"INSERT IGNORE INTO MsgTagTable (TagId, EntryNum)"
+ " SELECT TagTable.TagId, :{0} AS EntryNum FROM TagTable WHERE Tag = :{1}",
QueryType.CONNECT_TAG.getParamName(0),
QueryType.CONNECT_TAG.getParamName(1));
case FIND_MSG_ID:
return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId";
return MessageFormat.format(
"SELECT EntryNum From MsgTable WHERE MsgId = :{0}",
QueryType.FIND_MSG_ID.getParamName(0));
case FIND_TAG_ID:
return MessageFormat.format(
"SELECT TagId FROM TagTable WHERE Tag = :{0}",
QueryType.FIND_TAG_ID.getParamName(0));
case GET_MESSAGES:
return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable";
case COUNT_MESSAGES:
return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable";
case GET_MESSAGE_STUBS:
return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable";
case GET_SIGNATURES:
return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum";
return MessageFormat.format(
"SELECT Signature FROM SignatureTable WHERE EntryNum = :{0}",
QueryType.GET_SIGNATURES.getParamName(0));
case INSERT_MSG:
return "INSERT INTO MsgTable (MsgId, Msg) VALUES(:MsgId, :Msg)";
return MessageFormat.format(
"INSERT INTO MsgTable (MsgId, ExactTime, Msg) VALUES(:{0}, :{1}, :{2})",
QueryType.INSERT_MSG.getParamName(0),
QueryType.INSERT_MSG.getParamName(1),
QueryType.INSERT_MSG.getParamName(2));
case DELETE_MSG_BY_ENTRY:
return MessageFormat.format(
"DELETE IGNORE FROM MsgTable WHERE EntryNum = :{0}",
QueryType.DELETE_MSG_BY_ENTRY.getParamName(0));
case DELETE_MSG_BY_ID:
return MessageFormat.format(
"DELETE IGNORE FROM MsgTable WHERE MsgId = :{0}",
QueryType.DELETE_MSG_BY_ID.getParamName(0));
case INSERT_NEW_TAG:
return "INSERT IGNORE INTO TagTable(Tag) VALUES (:Tag)";
return MessageFormat.format(
"INSERT IGNORE INTO TagTable(Tag) VALUES (:{0})",
QueryType.INSERT_NEW_TAG.getParamName(0));
case GET_LAST_MESSAGE_ENTRY:
return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable";
case GET_BATCH_MESSAGE_DATA_BY_MSG_ID:
return MessageFormat.format(
"SELECT Data FROM BatchTable"
+ " INNER JOIN MsgTable ON MsgTable.EntryNum = BatchTable.EntryNum"
+ " WHERE MsgTable.MsgId = :{0} AND BatchTable.SerialNum >= :{1}"
+ " ORDER BY BatchTable.SerialNum ASC",
QueryType.GET_BATCH_MESSAGE_DATA_BY_MSG_ID.getParamName(0),
QueryType.GET_BATCH_MESSAGE_DATA_BY_MSG_ID.getParamName(1));
case GET_BATCH_MESSAGE_DATA_BY_BATCH_ID:
return MessageFormat.format(
"SELECT Data FROM BatchTable"
+ " WHERE BatchId = :{0} AND SerialNum >= :{1}"
+ " ORDER BY BatchTable.SerialNum ASC",
QueryType.GET_BATCH_MESSAGE_DATA_BY_BATCH_ID.getParamName(0),
QueryType.GET_BATCH_MESSAGE_DATA_BY_BATCH_ID.getParamName(1));
case INSERT_BATCH_DATA:
return MessageFormat.format(
"INSERT INTO BatchTable (BatchId, SerialNum, Data) VALUES (:{0}, :{1}, :{2})",
QueryType.INSERT_BATCH_DATA.getParamName(0),
QueryType.INSERT_BATCH_DATA.getParamName(1),
QueryType.INSERT_BATCH_DATA.getParamName(2));
case CHECK_BATCH_LENGTH:
return MessageFormat.format(
"SELECT COUNT(Data) AS BatchLength FROM BatchTable WHERE BatchId = :{0}",
QueryType.CHECK_BATCH_LENGTH.getParamName(0));
case CHECK_BATCH_OPEN:
return MessageFormat.format(
"SELECT COUNT(BatchId) AS batchCount FROM BatchTagTable WHERE BatchId = :{0}",
QueryType.CHECK_BATCH_OPEN.getParamName(0));
case STORE_BATCH_TAGS:
return MessageFormat.format(
"INSERT INTO BatchTagTable (Tags) VALUES (:{0})",
QueryType.STORE_BATCH_TAGS.getParamName(0));
case GET_BATCH_TAGS:
return MessageFormat.format(
"SELECT Tags FROM BatchTagTable WHERE BatchId = :{0}",
QueryType.GET_BATCH_TAGS.getParamName(0));
case ADD_ENTRY_NUM_TO_BATCH:
return MessageFormat.format(
"UPDATE BatchTable SET EntryNum = :{1} WHERE BatchId = :{0}",
QueryType.ADD_ENTRY_NUM_TO_BATCH.getParamName(0),
QueryType.ADD_ENTRY_NUM_TO_BATCH.getParamName(1));
default:
throw new IllegalArgumentException("Cannot serve a query of type " + queryType);
}
@ -63,6 +163,8 @@ public class MySQLQueryProvider implements SQLQueryProvider {
return "MsgTable.EntryNum = :EntryNum" + serialString;
case MAX_ENTRY:
return "MsgTable.EntryNum <= :EntryNum" + serialString;
case MIN_ENTRY:
return "MsgTable.EntryNum >= :EntryNum" + serialString;
case MAX_MESSAGES:
return "LIMIT :Limit" + serialString;
case MSG_ID:
@ -74,6 +176,13 @@ public class MySQLQueryProvider implements SQLQueryProvider {
return "EXISTS (SELECT 1 FROM TagTable"
+ " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId"
+ " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)";
case BEFORE_TIME:
return "MsgTable.ExactTime <= :TimeStamp";
case AFTER_TIME:
return "MsgTable.ExactTime >= :TimeStamp";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
@ -86,6 +195,7 @@ public class MySQLQueryProvider implements SQLQueryProvider {
switch(filterType) {
case EXACT_ENTRY: // Go through
case MAX_ENTRY: // Go through
case MIN_ENTRY: // Go through
case MAX_MESSAGES:
return "INT";
@ -96,6 +206,10 @@ public class MySQLQueryProvider implements SQLQueryProvider {
case TAG:
return "VARCHAR";
case AFTER_TIME: // Go through
case BEFORE_TIME:
return "TIMESTAMP";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
@ -104,32 +218,44 @@ public class MySQLQueryProvider implements SQLQueryProvider {
@Override
public DataSource getDataSource() {
MysqlDataSource dataSource = new MysqlDataSource();
dataSource.setServerName(dbAddress);
dataSource.setPort(dbPort);
dataSource.setDatabaseName(dbName);
dataSource.setUser(username);
BasicDataSource dataSource = new BasicDataSource();
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl("jdbc:mysql://" + dbAddress + ":" + dbPort + "/" + dbName);
dataSource.setUsername(username);
dataSource.setPassword(password);
return dataSource;
}
@Override
public List<String> getSchemaCreationCommands() {
List<String> list = new LinkedList<String>();
list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY, MsgId TINYBLOB, Msg BLOB, UNIQUE(MsgId(50)))");
list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INT NOT NULL AUTO_INCREMENT PRIMARY KEY,"
+ " MsgId TINYBLOB, ExactTime TIMESTAMP, Msg BLOB, UNIQUE(MsgId(50)))");
list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INT NOT NULL AUTO_INCREMENT PRIMARY KEY, Tag VARCHAR(50), UNIQUE(Tag))");
list.add("CREATE TABLE IF NOT EXISTS MsgTagTable (EntryNum INT, TagId INT,"
+ " CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum),"
+ " CONSTRAINT FOREIGN KEY (TagId) REFERENCES TagTable(TagId),"
+ " CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum) ON DELETE CASCADE,"
+ " CONSTRAINT FOREIGN KEY (TagId) REFERENCES TagTable(TagId) ON DELETE CASCADE,"
+ " CONSTRAINT UNIQUE (EntryNum, TagID))");
list.add("CREATE TABLE IF NOT EXISTS SignatureTable (EntryNum INT, SignerId TINYBLOB, Signature TINYBLOB,"
+ " INDEX(SignerId(32)), CONSTRAINT Uni UNIQUE(SignerId(32), EntryNum), CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))");
+ " INDEX(SignerId(32)), CONSTRAINT Unique_Signature UNIQUE(SignerId(32), EntryNum),"
+ " CONSTRAINT FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum) ON DELETE CASCADE)");
list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (BatchId INT AUTO_INCREMENT PRIMARY KEY, Tags BLOB)");
list.add("CREATE TABLE IF NOT EXISTS BatchTable (BatchId INT, EntryNum INT, SerialNum INT, Data BLOB,"
+ " CONSTRAINT UNIQUE(BatchId, SerialNum),"
+ " CONSTRAINT FOREIGN KEY (BatchId) REFERENCES BatchTagTable(BatchId) ON DELETE CASCADE)");
return list;
}
@ -138,6 +264,8 @@ public class MySQLQueryProvider implements SQLQueryProvider {
public List<String> getSchemaDeletionCommands() {
List<String> list = new LinkedList<String>();
list.add("DROP TABLE IF EXISTS BatchTable");
list.add("DROP TABLE IF EXISTS BatchTagTable");
list.add("DROP TABLE IF EXISTS MsgTagTable");
list.add("DROP TABLE IF EXISTS SignatureTable");
list.add("DROP TABLE IF EXISTS TagTable");
@ -145,4 +273,4 @@ public class MySQLQueryProvider implements SQLQueryProvider {
return list;
}
}
}

130
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/SQLiteQueryProvider.java
View File

@ -1,9 +1,11 @@
package meerkat.bulletinboard.sqlserver;
import meerkat.protobuf.BulletinBoardAPI.*;
import org.apache.commons.dbcp2.BasicDataSource;
import org.sqlite.SQLiteDataSource;
import javax.sql.DataSource;
import java.text.MessageFormat;
import java.util.LinkedList;
import java.util.List;
@ -25,19 +27,90 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi
switch(queryType) {
case ADD_SIGNATURE:
return "INSERT OR IGNORE INTO SignatureTable (EntryNum, SignerId, Signature) VALUES (:EntryNum,:SignerId,:Signature)";
case CONNECT_TAG:
return "INSERT OR IGNORE INTO MsgTagTable (TagId, EntryNum)"
+ " SELECT TagTable.TagId, :EntryNum AS EntryNum FROM TagTable WHERE Tag = :Tag";
case FIND_MSG_ID:
return "SELECT EntryNum From MsgTable WHERE MsgId = :MsgId";
case FIND_TAG_ID:
return MessageFormat.format(
"SELECT TagId FROM TagTable WHERE Tag = :{0}",
QueryType.FIND_TAG_ID.getParamName(0));
case GET_MESSAGES:
return "SELECT MsgTable.EntryNum, MsgTable.Msg FROM MsgTable";
case COUNT_MESSAGES:
return "SELECT COUNT(MsgTable.EntryNum) FROM MsgTable";
case GET_MESSAGE_STUBS:
return "SELECT MsgTable.EntryNum, MsgTable.MsgId, MsgTable.ExactTime FROM MsgTable";
case GET_SIGNATURES:
return "SELECT Signature FROM SignatureTable WHERE EntryNum = :EntryNum";
case INSERT_MSG:
return "INSERT INTO MsgTable (MsgId, Msg) VALUES(:MsgId,:Msg)";
case INSERT_NEW_TAG:
return "INSERT OR IGNORE INTO TagTable(Tag) VALUES (:Tag)";
case GET_LAST_MESSAGE_ENTRY:
return "SELECT MAX(MsgTable.EntryNum) FROM MsgTable";
case GET_BATCH_MESSAGE_DATA_BY_MSG_ID:
return MessageFormat.format(
"SELECT Data FROM BatchTable"
+ " INNER JOIN MsgTable ON MsgTable.EntryNum = BatchTable.EntryNum"
+ " WHERE MsgTable.MsgId = :{0} AND BatchTable.SerialNum >= :{1}"
+ " ORDER BY BatchTable.SerialNum ASC",
QueryType.GET_BATCH_MESSAGE_DATA_BY_MSG_ID.getParamName(0),
QueryType.GET_BATCH_MESSAGE_DATA_BY_MSG_ID.getParamName(1));
case GET_BATCH_MESSAGE_DATA_BY_BATCH_ID:
return MessageFormat.format(
"SELECT Data FROM BatchTable"
+ " WHERE BatchId = :{0} AND SerialNum >= :{1}"
+ " ORDER BY BatchTable.SerialNum ASC",
QueryType.GET_BATCH_MESSAGE_DATA_BY_BATCH_ID.getParamName(0),
QueryType.GET_BATCH_MESSAGE_DATA_BY_BATCH_ID.getParamName(1));
case INSERT_BATCH_DATA:
return MessageFormat.format(
"INSERT INTO BatchTable (BatchId, SerialNum, Data) VALUES (:{0}, :{1}, :{2})",
QueryType.INSERT_BATCH_DATA.getParamName(0),
QueryType.INSERT_BATCH_DATA.getParamName(1),
QueryType.INSERT_BATCH_DATA.getParamName(2));
case CHECK_BATCH_LENGTH:
return MessageFormat.format(
"SELECT COUNT(Data) AS BatchLength FROM BatchTable WHERE BatchId = :{0}",
QueryType.CHECK_BATCH_LENGTH.getParamName(0));
case CHECK_BATCH_OPEN:
return MessageFormat.format(
"SELECT COUNT(BatchId) AS batchCount FROM BatchTagTable WHERE BatchId = :{0}",
QueryType.CHECK_BATCH_OPEN.getParamName(0));
case STORE_BATCH_TAGS:
return MessageFormat.format(
"INSERT INTO BatchTagTable (Tags) VALUES (:{0})",
QueryType.STORE_BATCH_TAGS.getParamName(0));
case GET_BATCH_TAGS:
return MessageFormat.format(
"SELECT Tags FROM BatchTagTable WHERE BatchId = :{0}",
QueryType.GET_BATCH_TAGS.getParamName(0));
case ADD_ENTRY_NUM_TO_BATCH:
return MessageFormat.format(
"UPDATE BatchTable SET EntryNum = :{1} WHERE BatchId = :{0}",
QueryType.ADD_ENTRY_NUM_TO_BATCH.getParamName(0),
QueryType.ADD_ENTRY_NUM_TO_BATCH.getParamName(1));
default:
throw new IllegalArgumentException("Cannot serve a query of type " + queryType);
}
@ -52,19 +125,34 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi
switch(filterType) {
case EXACT_ENTRY:
return "MsgTable.EntryNum = :EntryNum" + serialString;
case MAX_ENTRY:
return "MsgTable.EntryNum <= :EntryNum" + serialString;
case MIN_ENTRY:
return "MsgTable.EntryNum <= :EntryNum" + serialString;
case MAX_MESSAGES:
return "LIMIT = :Limit" + serialString;
case MSG_ID:
return "MsgTable.MsgId = :MsgId" + serialString;
case SIGNER_ID:
return "EXISTS (SELECT 1 FROM SignatureTable"
+ " WHERE SignatureTable.SignerId = :SignerId" + serialString + " AND SignatureTable.EntryNum = MsgTable.EntryNum)";
case TAG:
return "EXISTS (SELECT 1 FROM TagTable"
+ " INNER JOIN MsgTagTable ON TagTable.TagId = MsgTagTable.TagId"
+ " WHERE TagTable.Tag = :Tag" + serialString + " AND MsgTagTable.EntryNum = MsgTable.EntryNum)";
case BEFORE_TIME:
return "MsgTable.ExactTime <= :TimeStamp";
case AFTER_TIME:
return "MsgTable.ExactTime >= :TimeStamp";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
@ -73,15 +161,33 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi
@Override
public String getConditionParamTypeName(FilterType filterType) throws IllegalArgumentException {
return null; //TODO: write this.
switch(filterType) {
case EXACT_ENTRY: // Go through
case MAX_ENTRY: // Go through
case MIN_ENTRY: // Go through
case MAX_MESSAGES:
return "INTEGER";
case MSG_ID: // Go through
case SIGNER_ID:
return "BLOB";
case TAG:
return "VARCHAR";
default:
throw new IllegalArgumentException("Cannot serve a filter of type " + filterType);
}
}
@Override
public DataSource getDataSource() {
// TODO: Fix this
SQLiteDataSource dataSource = new SQLiteDataSource();
BasicDataSource dataSource = new BasicDataSource();
dataSource.setDriverClassName("org.sqlite.JDBC");
dataSource.setUrl("jdbc:sqlite:" + dbName);
dataSource.setDatabaseName("meerkat"); //TODO: Make generic
return dataSource;
}
@ -94,14 +200,22 @@ public class SQLiteQueryProvider implements BulletinBoardSQLServer.SQLQueryProvi
list.add("CREATE TABLE IF NOT EXISTS MsgTable (EntryNum INTEGER PRIMARY KEY, MsgId BLOB UNIQUE, Msg BLOB)");
list.add("CREATE TABLE IF NOT EXISTS TagTable (TagId INTEGER PRIMARY KEY, Tag varchar(50) UNIQUE)");
list.add("CREATE TABLE IF NOT EXISTS MsgTagTable (EntryNum BLOB, TagId INTEGER, FOREIGN KEY (EntryNum)"
+ " REFERENCES MsgTable(EntryNum), FOREIGN KEY (TagId) REFERENCES TagTable(TagId), UNIQUE (EntryNum, TagID))");
list.add("CREATE TABLE IF NOT EXISTS MsgTagTable (EntryNum BLOB, TagId INTEGER,"
+ " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum) ON DELETE CASCADE,"
+ " FOREIGN KEY (TagId) REFERENCES TagTable(TagId) ON DELETE CASCADE,"
+ " UNIQUE (EntryNum, TagID))");
list.add("CREATE TABLE IF NOT EXISTS SignatureTable (EntryNum INTEGER, SignerId BLOB, Signature BLOB,"
+ " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum))");
+ " FOREIGN KEY (EntryNum) REFERENCES MsgTable(EntryNum) ON DELETE CASCADE,"
+ " UNIQUE(SignerId, EntryNum))");
list.add("CREATE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId)");
list.add("CREATE UNIQUE INDEX IF NOT EXISTS SignerIndex ON SignatureTable(SignerId, EntryNum)");
list.add("CREATE TABLE IF NOT EXISTS BatchTagTable (BatchId INTEGER PRIMARY KEY, Tags BLOB)");
list.add("CREATE TABLE IF NOT EXISTS BatchTable (BatchId INTEGER, EntryNum INTEGER, SerialNum INTEGER, Data BLOB,"
+ " UNIQUE(BatchId, SerialNum),"
+ " FOREIGN KEY (BatchId) REFERENCES BatchTagTable(BatchId) ON DELETE CASCADE)");
return list;
}

30
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataCallbackHandler.java Normal file
View File

@ -0,0 +1,30 @@
package meerkat.bulletinboard.sqlserver.mappers;
import meerkat.comm.MessageOutputStream;
import meerkat.protobuf.BulletinBoardAPI.BatchChunk;
import org.springframework.jdbc.core.RowCallbackHandler;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 19-Dec-15.
*/
public class BatchDataCallbackHandler implements RowCallbackHandler {
private final MessageOutputStream<BatchChunk> out;
public BatchDataCallbackHandler(MessageOutputStream<BatchChunk> out) {
this.out = out;
}
@Override
public void processRow(ResultSet rs) throws SQLException {
try {
out.writeMessage(BatchChunk.parseFrom(rs.getBytes(1)));
} catch (IOException e) {
//TODO: Log
}
}
}

31
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BatchDataDigestHandler.java Normal file
View File

@ -0,0 +1,31 @@
package meerkat.bulletinboard.sqlserver.mappers;
import meerkat.bulletinboard.BulletinBoardDigest;
import meerkat.protobuf.BulletinBoardAPI.BatchChunk;
import org.springframework.jdbc.core.RowCallbackHandler;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 19-Dec-15.
*/
public class BatchDataDigestHandler implements RowCallbackHandler {
private final BulletinBoardDigest digest;
public BatchDataDigestHandler(BulletinBoardDigest digest) {
this.digest = digest;
}
@Override
public void processRow(ResultSet rs) throws SQLException {
try {
BatchChunk batchChunk = BatchChunk.parseFrom(rs.getBytes(1));
digest.update(batchChunk.getData().toByteArray());
} catch (IOException e) {
//TODO: Log
}
}
}

24
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/BeginBatchMessageMapper.java Normal file
View File

@ -0,0 +1,24 @@
package meerkat.bulletinboard.sqlserver.mappers;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.protobuf.BulletinBoardAPI.BeginBatchMessage;
import org.springframework.jdbc.core.RowMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 20-Dec-15.
*/
public class BeginBatchMessageMapper implements RowMapper<BeginBatchMessage> {
@Override
public BeginBatchMessage mapRow(ResultSet rs, int rowNum) throws SQLException {
try {
return BeginBatchMessage.newBuilder().mergeFrom(rs.getBytes(1)).build();
} catch (InvalidProtocolBufferException e) {
return null;
}
}
}

2
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/EntryNumMapper.java → bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/LongMapper.java
View File

@ -9,7 +9,7 @@ import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 11-Dec-15.
*/
public class EntryNumMapper implements RowMapper<Long> {
public class LongMapper implements RowMapper<Long> {
@Override
public Long mapRow(ResultSet rs, int rowNum) throws SQLException {

80
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageCallbackHandler.java Normal file
View File

@ -0,0 +1,80 @@
package meerkat.bulletinboard.sqlserver.mappers;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.*;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider.*;
import meerkat.comm.MessageOutputStream;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Crypto;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 21-Feb-16.
*/
public class MessageCallbackHandler implements RowCallbackHandler {
private final NamedParameterJdbcTemplate jdbcTemplate;
private final SQLQueryProvider sqlQueryProvider;
private final MessageOutputStream<BulletinBoardMessage> out;
public MessageCallbackHandler(NamedParameterJdbcTemplate jdbcTemplate, SQLQueryProvider sqlQueryProvider, MessageOutputStream<BulletinBoardMessage> out) {
this.jdbcTemplate = jdbcTemplate;
this.sqlQueryProvider = sqlQueryProvider;
this.out = out;
}
@Override
public void processRow(ResultSet rs) throws SQLException {
BulletinBoardMessage.Builder result;
try {
result = BulletinBoardMessage.newBuilder()
.setEntryNum(rs.getLong(1))
.setMsg(UnsignedBulletinBoardMessage.parseFrom(rs.getBytes(2)));
} catch (InvalidProtocolBufferException e) {
//TODO: log
return;
}
// Retrieve signatures
MapSqlParameterSource sqlParameterSource = new MapSqlParameterSource();
sqlParameterSource.addValue(QueryType.GET_SIGNATURES.getParamName(0), result.getEntryNum());
List<Crypto.Signature> signatures = jdbcTemplate.query(
sqlQueryProvider.getSQLString(QueryType.GET_SIGNATURES),
sqlParameterSource,
new SignatureMapper());
// Append signatures
result.addAllSig(signatures);
// Finalize message and add to message list.
try {
out.writeMessage(result.build());
} catch (IOException e) {
//TODO: log
e.printStackTrace();
}
}
}

59
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubCallbackHandler.java Normal file
View File

@ -0,0 +1,59 @@
package meerkat.bulletinboard.sqlserver.mappers;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer.SQLQueryProvider.QueryType;
import meerkat.comm.MessageOutputStream;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.protobuf.BulletinBoardAPI.UnsignedBulletinBoardMessage;
import meerkat.protobuf.Crypto;
import meerkat.util.BulletinBoardUtils;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
/**
* Created by Arbel Deutsch Peled on 21-Feb-16.
*/
public class MessageStubCallbackHandler implements RowCallbackHandler {
private final MessageOutputStream<BulletinBoardMessage> out;
public MessageStubCallbackHandler(MessageOutputStream<BulletinBoardMessage> out) {
this.out = out;
}
@Override
public void processRow(ResultSet rs) throws SQLException {
BulletinBoardMessage result;
result = BulletinBoardMessage.newBuilder()
.setEntryNum(rs.getLong(1))
.setMsg(UnsignedBulletinBoardMessage.newBuilder()
.setData(ByteString.copyFrom(rs.getBytes(2)))
.setTimestamp(BulletinBoardUtils.toTimestampProto(rs.getTimestamp(3)))
.build())
.build();
try {
out.writeMessage(result);
} catch (IOException e) {
//TODO: log
e.printStackTrace();
}
}
}

31
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/MessageStubMapper.java Normal file
View File

@ -0,0 +1,31 @@
package meerkat.bulletinboard.sqlserver.mappers;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.protobuf.BulletinBoardAPI.UnsignedBulletinBoardMessage;
import meerkat.util.BulletinBoardUtils;
import org.springframework.jdbc.core.RowMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 11-Dec-15.
*/
public class MessageStubMapper implements RowMapper<BulletinBoardMessage> {
@Override
public BulletinBoardMessage mapRow(ResultSet rs, int rowNum) throws SQLException {
return BulletinBoardMessage.newBuilder()
.setEntryNum(rs.getLong(1))
.setMsg(UnsignedBulletinBoardMessage.newBuilder()
.setMsgId(ByteString.copyFrom(rs.getBytes(2)))
.setTimestamp(BulletinBoardUtils.toTimestampProto(rs.getTimestamp(3)))
.build())
.build();
}
}

18
bulletin-board-server/src/main/java/meerkat/bulletinboard/sqlserver/mappers/StringMapper.java Normal file
View File

@ -0,0 +1,18 @@
package meerkat.bulletinboard.sqlserver.mappers;
import org.springframework.jdbc.core.RowMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Arbel Deutsch Peled on 20-Dec-15.
*/
public class StringMapper implements RowMapper<String> {
@Override
public String mapRow(ResultSet rs, int rowNum) throws SQLException {
return rs.getString(1);
}
}

208
bulletin-board-server/src/main/java/meerkat/bulletinboard/webapp/BulletinBoardWebApp.java
View File

@ -3,27 +3,32 @@ package meerkat.bulletinboard.webapp;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.StreamingOutput;
import com.google.protobuf.BoolValue;
import com.google.protobuf.Int32Value;
import com.google.protobuf.Int64Value;
import meerkat.bulletinboard.BulletinBoardServer;
import meerkat.bulletinboard.sqlserver.BulletinBoardSQLServer;
import meerkat.bulletinboard.sqlserver.H2QueryProvider;
import meerkat.bulletinboard.sqlserver.MySQLQueryProvider;
import meerkat.bulletinboard.sqlserver.SQLiteQueryProvider;
import meerkat.comm.CommunicationException;
import meerkat.protobuf.BulletinBoardAPI.BoolMsg;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessageList;
import meerkat.protobuf.BulletinBoardAPI.MessageFilterList;
import meerkat.rest.Constants;
import meerkat.comm.MessageOutputStream;
import meerkat.protobuf.BulletinBoardAPI.*;
import static meerkat.bulletinboard.BulletinBoardConstants.*;
import static meerkat.rest.Constants.*;
@Path(Constants.BULLETIN_BOARD_SERVER_PATH)
import java.io.IOException;
import java.io.OutputStream;
/**
* An implementation of the BulletinBoardServer which functions as a WebApp
*/
@Path(BULLETIN_BOARD_SERVER_PATH)
public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextListener{
private static final String BULLETIN_BOARD_ATTRIBUTE_NAME = "bulletinBoard";
@ -39,14 +44,6 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL
bulletinBoard = (BulletinBoardServer) servletContext.getAttribute(BULLETIN_BOARD_ATTRIBUTE_NAME);
}
/**
* This is the BulletinBoard init method.
*/
@Override
public void init(String meerkatDB) throws CommunicationException {
bulletinBoard.init(meerkatDB);
}
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
ServletContext servletContext = servletContextEvent.getServletContext();
@ -72,31 +69,178 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL
}
try {
init(dbName);
bulletinBoard.init();
servletContext.setAttribute(BULLETIN_BOARD_ATTRIBUTE_NAME, bulletinBoard);
} catch (CommunicationException e) {
System.err.println(e.getMessage());
}
}
@Path(Constants.POST_MESSAGE_PATH)
@Path(POST_MESSAGE_PATH)
@POST
@Consumes(Constants.MEDIATYPE_PROTOBUF)
@Produces(Constants.MEDIATYPE_PROTOBUF)
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public BoolMsg postMessage(BulletinBoardMessage msg) throws CommunicationException {
public BoolValue postMessage(BulletinBoardMessage msg) throws CommunicationException {
init();
return bulletinBoard.postMessage(msg);
}
@Path(Constants.READ_MESSAGES_PATH)
@POST
@Consumes(Constants.MEDIATYPE_PROTOBUF)
@Produces(Constants.MEDIATYPE_PROTOBUF)
@Override
public BulletinBoardMessageList readMessages(MessageFilterList filterList) throws CommunicationException {
public void readMessages(MessageFilterList filterList, MessageOutputStream<BulletinBoardMessage> out) throws CommunicationException {
init();
return bulletinBoard.readMessages(filterList);
bulletinBoard.readMessages(filterList, out);
}
@Path(COUNT_MESSAGES_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public Int32Value getMessageCount(MessageFilterList filterList) throws CommunicationException {
init();
return bulletinBoard.getMessageCount(filterList);
}
@Path(READ_MESSAGES_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
/**
* Wrapper for the readMessages method which streams the output into the response
*/
public StreamingOutput readMessages(final MessageFilterList filterList) {
return new StreamingOutput() {
@Override
public void write(OutputStream output) throws IOException, WebApplicationException {
MessageOutputStream<BulletinBoardMessage> out = new MessageOutputStream<>(output);
try {
init();
bulletinBoard.readMessages(filterList, out);
} catch (CommunicationException e) {
//TODO: Log
out.writeMessage(null);
}
}
};
}
@Path(BEGIN_BATCH_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public Int64Value beginBatch(BeginBatchMessage message) {
try {
init();
return bulletinBoard.beginBatch(message);
} catch (CommunicationException e) {
System.err.println(e.getMessage());
return null;
}
}
@Path(POST_BATCH_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public BoolValue postBatchMessage(BatchMessage batchMessage) {
try {
init();
return bulletinBoard.postBatchMessage(batchMessage);
} catch (CommunicationException e) {
System.err.println(e.getMessage());
return null;
}
}
@Path(CLOSE_BATCH_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public BoolValue closeBatch(CloseBatchMessage message) {
try {
init();
return bulletinBoard.closeBatch(message);
} catch (CommunicationException e) {
System.err.println(e.getMessage());
return null;
}
}
@Override
public void readBatch(BatchQuery batchQuery, MessageOutputStream<BatchChunk> out) throws CommunicationException, IllegalArgumentException {
try {
init();
bulletinBoard.readBatch(batchQuery, out);
} catch (CommunicationException | IllegalArgumentException e) {
System.err.println(e.getMessage());
}
}
@Path(GENERATE_SYNC_QUERY_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public SyncQuery generateSyncQuery(GenerateSyncQueryParams generateSyncQueryParams) throws CommunicationException {
try {
init();
return bulletinBoard.generateSyncQuery(generateSyncQueryParams);
} catch (CommunicationException | IllegalArgumentException e) {
System.err.println(e.getMessage());
return null;
}
}
@Path(READ_BATCH_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
/**
* Wrapper for the readBatch method which streams the output into the response
*/
public StreamingOutput readBatch(final BatchQuery batchQuery) {
return new StreamingOutput() {
@Override
public void write(OutputStream output) throws IOException, WebApplicationException {
MessageOutputStream<BatchChunk> out = new MessageOutputStream<>(output);
try {
init();
bulletinBoard.readBatch(batchQuery, out);
} catch (CommunicationException e) {
//TODO: Log
out.writeMessage(null);
}
}
};
}
@Path(SYNC_QUERY_PATH)
@POST
@Consumes(MEDIATYPE_PROTOBUF)
@Produces(MEDIATYPE_PROTOBUF)
@Override
public SyncQueryResponse querySync(SyncQuery syncQuery) throws CommunicationException {
try{
init();
return bulletinBoard.querySync(syncQuery);
} catch (CommunicationException | IllegalArgumentException e) {
System.err.println(e.getMessage());
return null;
}
}
@Override
@ -121,4 +265,4 @@ public class BulletinBoardWebApp implements BulletinBoardServer, ServletContextL
close();
}
}
}

9
bulletin-board-server/src/main/proto/meerkat/bulletin_board_server.proto
View File

@ -1,9 +0,0 @@
syntax = "proto3";
package meerkat;
option java_package = "meerkat.protobuf";
message Boolean {
bool value = 1;
}

2
bulletin-board-server/src/main/webapp/WEB-INF/web.xml
View File

@ -31,7 +31,7 @@
<param-value>mypass</param-value></context-param>
<context-param>
<param-name>dbType</param-name>
<param-value>SQLite</param-value></context-param>
<param-value>H2</param-value></context-param>
<listener>
<listener-class>meerkat.bulletinboard.webapp.BulletinBoardWebApp</listener-class>
</listener>

54
bulletin-board-server/src/test/java/meerkat/bulletinboard/BulletinBoardSQLServerIntegrationTest.java
View File

@ -1,11 +1,16 @@
package meerkat.bulletinboard;
import com.google.protobuf.BoolValue;
import com.google.protobuf.ByteString;
import com.google.protobuf.TextFormat;
import com.google.protobuf.Timestamp;
import meerkat.comm.MessageInputStream;
import meerkat.protobuf.Crypto.*;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.protobuf.Comm.*;
import static meerkat.bulletinboard.BulletinBoardConstants.*;
import meerkat.rest.Constants;
import meerkat.rest.ProtobufMessageBodyReader;
import meerkat.rest.ProtobufMessageBodyWriter;
@ -18,6 +23,8 @@ import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import java.io.InputStream;
import java.util.List;
public class BulletinBoardSQLServerIntegrationTest {
@ -43,19 +50,29 @@ public class BulletinBoardSQLServerIntegrationTest {
byte[] b3 = {(byte) 21, (byte) 22, (byte) 23, (byte) 24};
byte[] b4 = {(byte) 4, (byte) 5, (byte) 100, (byte) -50, (byte) 0};
Timestamp t1 = Timestamp.newBuilder()
.setSeconds(8276482)
.setNanos(4314)
.build();
Timestamp t2 = Timestamp.newBuilder()
.setSeconds(987591)
.setNanos(1513)
.build();
WebTarget webTarget;
Response response;
BoolMsg bool;
BoolValue bool;
BulletinBoardMessage msg;
MessageFilterList filterList;
BulletinBoardMessageList msgList;
List<BulletinBoardMessage> msgList;
// Test writing mechanism
System.err.println("******** Testing: " + Constants.POST_MESSAGE_PATH);
webTarget = client.target(BASE_URL).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.POST_MESSAGE_PATH);
System.err.println("******** Testing: " + POST_MESSAGE_PATH);
webTarget = client.target(BASE_URL).path(BULLETIN_BOARD_SERVER_PATH).path(POST_MESSAGE_PATH);
System.err.println(webTarget.getUri());
msg = BulletinBoardMessage.newBuilder()
@ -63,6 +80,7 @@ public class BulletinBoardSQLServerIntegrationTest {
.addTag("Signature")
.addTag("Trustee")
.setData(ByteString.copyFrom(b1))
.setTimestamp(t1)
.build())
.addSig(Signature.newBuilder()
.setType(SignatureType.DSA)
@ -78,7 +96,7 @@ public class BulletinBoardSQLServerIntegrationTest {
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF));
System.err.println(response);
bool = response.readEntity(BoolMsg.class);
bool = response.readEntity(BoolValue.class);
assert bool.getValue();
msg = BulletinBoardMessage.newBuilder()
@ -86,6 +104,7 @@ public class BulletinBoardSQLServerIntegrationTest {
.addTag("Vote")
.addTag("Trustee")
.setData(ByteString.copyFrom(b4))
.setTimestamp(t2)
.build())
.addSig(Signature.newBuilder()
.setType(SignatureType.ECDSA)
@ -96,13 +115,13 @@ public class BulletinBoardSQLServerIntegrationTest {
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(msg, Constants.MEDIATYPE_PROTOBUF));
System.err.println(response);
bool = response.readEntity(BoolMsg.class);
bool = response.readEntity(BoolValue.class);
assert bool.getValue();
// Test reading mechanism
System.err.println("******** Testing: " + Constants.READ_MESSAGES_PATH);
webTarget = client.target(BASE_URL).path(Constants.BULLETIN_BOARD_SERVER_PATH).path(Constants.READ_MESSAGES_PATH);
System.err.println("******** Testing: " + READ_MESSAGES_PATH);
webTarget = client.target(BASE_URL).path(BULLETIN_BOARD_SERVER_PATH).path(READ_MESSAGES_PATH);
filterList = MessageFilterList.newBuilder()
.addFilter(
MessageFilter.newBuilder()
@ -112,13 +131,20 @@ public class BulletinBoardSQLServerIntegrationTest {
)
.build();
response = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF));
System.err.println(response);
msgList = response.readEntity(BulletinBoardMessageList.class);
System.err.println("List size: " + msgList.getMessageCount());
InputStream in = webTarget.request(Constants.MEDIATYPE_PROTOBUF).post(Entity.entity(filterList, Constants.MEDIATYPE_PROTOBUF), InputStream.class);
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStream.MessageInputStreamFactory.createMessageInputStream(in, BulletinBoardMessage.class);
msgList = inputStream.asList();
System.err.println("List size: " + msgList.size());
System.err.println("This is the list:");
System.err.println(TextFormat.printToString(msgList));
assert msgList.getMessageCount() == 1;
for (BulletinBoardMessage message : msgList) {
System.err.println(TextFormat.printToString(message));
}
assert msgList.size() == 1;
}
}

340
bulletin-board-server/src/test/java/meerkat/bulletinboard/GenericBulletinBoardServerTest.java
View File

@ -1,9 +1,12 @@
package meerkat.bulletinboard;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.lang.reflect.InvocationTargetException;
import java.math.BigInteger;
import java.security.InvalidKeyException;
import java.security.KeyStore;
@ -12,26 +15,32 @@ import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.util.List;
import java.util.Random;
import java.util.*;
import com.google.protobuf.BoolValue;
import com.google.protobuf.ByteString;
import com.google.protobuf.Int64Value;
import com.google.protobuf.Timestamp;
import meerkat.comm.CommunicationException;
import meerkat.comm.MessageInputStream;
import meerkat.comm.MessageOutputStream;
import meerkat.comm.MessageInputStream.MessageInputStreamFactory;
import meerkat.crypto.concrete.ECDSASignature;
import meerkat.protobuf.BulletinBoardAPI.BulletinBoardMessage;
import meerkat.protobuf.BulletinBoardAPI.FilterType;
import meerkat.protobuf.BulletinBoardAPI.MessageFilter;
import meerkat.protobuf.BulletinBoardAPI.MessageFilterList;
import meerkat.protobuf.BulletinBoardAPI.UnsignedBulletinBoardMessage;
import meerkat.crypto.concrete.SHA256Digest;
import meerkat.protobuf.BulletinBoardAPI.*;
import meerkat.util.BulletinBoardMessageComparator;
import meerkat.util.BulletinBoardMessageGenerator;
import meerkat.util.BulletinBoardUtils;
import static org.junit.Assert.*;
import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.MatcherAssert.assertThat;
public class GenericBulletinBoardServerTest {
protected BulletinBoardServer bulletinBoardServer;
private ECDSASignature signers[];
private GenericBulletinBoardSignature[] signers;
private ByteString[] signerIDs;
private Random random;
@ -51,18 +60,18 @@ public class GenericBulletinBoardServerTest {
private String[] tags;
private byte[][] data;
private List<BulletinBoardMessage> batches;
private final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); // Used to time the tests
private BulletinBoardMessageGenerator bulletinBoardMessageGenerator;
private BulletinBoardDigest digest;
private BulletinBoardMessageComparator comparator;
/**
* @param bulletinBoardServer is an initialized server.
* @throws InstantiationException
* @throws IllegalAccessException
* @throws CertificateException
* @throws KeyStoreException
* @throws NoSuchAlgorithmException
* @throws IOException
* @throws UnrecoverableKeyException
* @throws CommunicationException
*/
public void init(BulletinBoardServer bulletinBoardServer) {
@ -71,10 +80,10 @@ public class GenericBulletinBoardServerTest {
this.bulletinBoardServer = bulletinBoardServer;
signers = new ECDSASignature[2];
signers = new GenericBulletinBoardSignature[2];
signerIDs = new ByteString[signers.length];
signers[0] = new ECDSASignature();
signers[1] = new ECDSASignature();
signers[0] = new GenericBulletinBoardSignature(new ECDSASignature());
signers[1] = new GenericBulletinBoardSignature(new ECDSASignature());
InputStream keyStream = getClass().getResourceAsStream(KEYFILE_EXAMPLE);
char[] password = KEYFILE_PASSWORD1.toCharArray();
@ -115,12 +124,23 @@ public class GenericBulletinBoardServerTest {
System.err.println("Couldn't find signing key " + e.getMessage());
fail("Couldn't find signing key " + e.getMessage());
}
random = new Random(0); // We use insecure randomness in tests for repeatability
// We use insecure randomness in tests for repeatability
random = new Random(0);
bulletinBoardMessageGenerator = new BulletinBoardMessageGenerator(random);
digest = new GenericBulletinBoardDigest(new SHA256Digest());
comparator = new BulletinBoardMessageComparator();
long end = threadBean.getCurrentThreadCpuTime();
System.err.println("Finished initializing GenericBulletinBoardServerTest");
System.err.println("Time of operation: " + (end - start));
// Initialize Batch variables
batches = new ArrayList<>(10);
}
private byte randomByte(){
@ -170,7 +190,11 @@ public class GenericBulletinBoardServerTest {
for (i = 1; i <= MESSAGE_NUM; i++) {
unsignedMsgBuilder = UnsignedBulletinBoardMessage.newBuilder()
.setData(ByteString.copyFrom(data[i - 1]));
.setData(ByteString.copyFrom(data[i - 1]))
.setTimestamp(Timestamp.newBuilder()
.setSeconds(i)
.setNanos(i)
.build());
// Add tags based on bit-representation of message number.
@ -230,28 +254,39 @@ public class GenericBulletinBoardServerTest {
System.err.println("Starting to test tag and signature mechanism");
long start = threadBean.getCurrentThreadCpuTime();
List<BulletinBoardMessage> messages;
List<BulletinBoardMessage> messages = new LinkedList<>();
// Check tag mechanism
for (int i = 0 ; i < TAG_NUM ; i++){
// Retrieve messages having tag i
try {
messages = bulletinBoardServer.readMessages(
MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag(tags[i])
.build()
)
.build()
)
.getMessageList();
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.TAG)
.setTag(tags[i])
.build()
)
.build();
} catch (CommunicationException e) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
bulletinBoardServer.readMessages(filterList, new MessageOutputStream<BulletinBoardMessage>(outputStream));
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BulletinBoardMessage.class);
messages = inputStream.asList();
} catch (CommunicationException | IOException e) {
fail(e.getMessage());
return;
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
fail(e.getMessage());
return;
}
@ -328,11 +363,26 @@ public class GenericBulletinBoardServerTest {
);
try {
messages = bulletinBoardServer.readMessages(filterListBuilder.build()).getMessageList();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
bulletinBoardServer.readMessages(filterListBuilder.build(), new MessageOutputStream<BulletinBoardMessage>(outputStream));
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BulletinBoardMessage.class);
messages = inputStream.asList();
} catch (CommunicationException e) {
System.err.println("Failed retrieving multi-tag messages from DB: " + e.getMessage());
fail("Failed retrieving multi-tag messages from DB: " + e.getMessage());
return;
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | IOException e) {
System.err.println("Falied to read from stream while retrieving multi-tag messages: " + e.getMessage());
fail("Falied to read from stream while retrieving multi-tag messages: " + e.getMessage());
return;
}
expectedMsgCount /= 2;
@ -359,11 +409,26 @@ public class GenericBulletinBoardServerTest {
.build());
try {
messages = bulletinBoardServer.readMessages(filterListBuilder.build()).getMessageList();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
bulletinBoardServer.readMessages(filterListBuilder.build(), new MessageOutputStream<BulletinBoardMessage>(outputStream));
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BulletinBoardMessage.class);
messages = inputStream.asList();
} catch (CommunicationException e) {
System.err.println("Failed retrieving multi-signature message from DB: " + e.getMessage());
fail("Failed retrieving multi-signature message from DB: " + e.getMessage());
return;
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | IOException e) {
System.err.println("Falied to read from stream while retrieving multi-signature message: " + e.getMessage());
fail("Falied to read from stream while retrieving multi-signature message: " + e.getMessage());
return;
}
assertThat(messages.size(), is(MESSAGE_NUM / 4));
@ -377,6 +442,207 @@ public class GenericBulletinBoardServerTest {
System.err.println("Time of operation: " + (end - start));
}
private void postAsBatch(BulletinBoardMessage message, int chunkSize, boolean close) throws CommunicationException {
List<BatchChunk> batchChunks = BulletinBoardUtils.breakToBatch(message, chunkSize);
BeginBatchMessage beginBatchMessage = BulletinBoardUtils.generateBeginBatchMessage(message);
BoolValue result;
// Begin batch
Int64Value batchId = bulletinBoardServer.beginBatch(beginBatchMessage);
assertThat("Was not able to open batch", batchId.getValue() != -1);
// Post data
BatchMessage batchMessage = BatchMessage.getDefaultInstance();
for (int i = 0 ; i < batchChunks.size() ; i++){
batchMessage = BatchMessage.newBuilder()
.setBatchId(batchId.getValue())
.setSerialNum(i)
.setData(batchChunks.get(i))
.build();
result = bulletinBoardServer.postBatchMessage(batchMessage);
assertThat("Was not able to post batch message", result.getValue(), is(true));
}
// Close batch
if (close) {
CloseBatchMessage closeBatchMessage = BulletinBoardUtils.generateCloseBatchMessage(batchId, batchChunks.size(), message);
result = bulletinBoardServer.closeBatch(closeBatchMessage);
assertThat("Was not able to close batch", result.getValue(), is(true));
}
}
/**
* Posts a complete batch message
* @throws CommunicationException
*/
public void testPostBatch() throws CommunicationException, SignatureException {
// Create data
final int BATCH_ID = 200;
final int DATA_SIZE = 10000;
final int CHUNK_SIZE = 100;
final int TAG_NUMBER = 10;
Timestamp timestamp = Timestamp.newBuilder()
.setSeconds(5235000)
.setNanos(32541)
.build();
BulletinBoardMessage batch = bulletinBoardMessageGenerator.generateRandomMessage(signers, timestamp, DATA_SIZE, TAG_NUMBER);
// Post batch
postAsBatch(batch, CHUNK_SIZE, true);
// Update locally stored batches
batches.add(batch);
}
public void testReadBatch() throws CommunicationException {
for (BulletinBoardMessage message : batches) {
try {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
digest.update(message);
MessageID msgId = digest.digestAsMessageID();
MessageFilterList messageFilterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(msgId.getID())
.build())
.build();
bulletinBoardServer.readMessages(messageFilterList, new MessageOutputStream<BulletinBoardMessage>(outputStream));
MessageInputStream<BulletinBoardMessage> messageInputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BulletinBoardMessage.class);
List<BulletinBoardMessage> messageList = messageInputStream.asList();
assertThat("No stub found for message ID " + msgId.getID().toStringUtf8(), messageList.size() == 1);
BulletinBoardMessage stub = messageList.get(0);
BatchQuery batchQuery =
BatchQuery.newBuilder()
.setMsgID(msgId)
.setStartPosition(0)
.build();
bulletinBoardServer.readBatch(batchQuery, new MessageOutputStream<BatchChunk>(outputStream));
MessageInputStream<BatchChunk> batchInputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BatchChunk.class);
List<BatchChunk> batchChunkList = batchInputStream.asList();
BulletinBoardMessage retrievedMessage = BulletinBoardUtils.gatherBatch(stub, batchChunkList);
assertThat("Non-matching batch data for batch " + msgId.getID().toStringUtf8(),
comparator.compare(message, retrievedMessage) == 0);
} catch (IOException | NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
assertThat("Error reading batch data list from input stream", false);
}
}
}
public void testSyncQuery()
throws SignatureException, CommunicationException, IOException,NoSuchMethodException, IllegalAccessException, InvocationTargetException {
Checksum checksum = new SimpleChecksum();
Timestamp timestamp = Timestamp.newBuilder()
.setSeconds(1)
.setNanos(0)
.build();
BulletinBoardMessage newMessage = bulletinBoardMessageGenerator.generateRandomMessage(signers, timestamp, 10, 10);
BoolValue result = bulletinBoardServer.postMessage(newMessage);
assertThat("Failed to post message to BB Server", result.getValue(), is(true));
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
digest.update(newMessage.getMsg());
ByteString messageID = ByteString.copyFrom(digest.digest());
MessageFilterList filterList = MessageFilterList.newBuilder()
.addFilter(MessageFilter.newBuilder()
.setType(FilterType.MSG_ID)
.setId(messageID)
.build())
.build();
bulletinBoardServer.readMessages(filterList, new MessageOutputStream<BulletinBoardMessage>(outputStream));
MessageInputStream<BulletinBoardMessage> inputStream =
MessageInputStreamFactory.createMessageInputStream(new ByteArrayInputStream(
outputStream.toByteArray()),
BulletinBoardMessage.class);
long lastEntry = inputStream.asList().get(0).getEntryNum();
SyncQuery syncQuery = SyncQuery.newBuilder()
.setFilterList(MessageFilterList.getDefaultInstance())
.addQuery(SingleSyncQuery.newBuilder()
.setChecksum(2)
.setTimeOfSync(Timestamp.newBuilder()
.setSeconds(2)
.setNanos(0)
.build())
.build())
.build();
SyncQueryResponse queryResponse = bulletinBoardServer.querySync(syncQuery);
assertThat("Sync query replies with positive sync when no sync was expected", queryResponse.getLastEntryNum(), is(equalTo(-1l)));
syncQuery = SyncQuery.newBuilder()
.setFilterList(MessageFilterList.getDefaultInstance())
.addQuery(SingleSyncQuery.newBuilder()
.setChecksum(checksum.getChecksum(messageID))
.setTimeOfSync(timestamp)
.build())
.build();
queryResponse = bulletinBoardServer.querySync(syncQuery);
assertThat("Sync query reply contained wrong last entry number", lastEntry, is(equalTo(queryResponse.getLastEntryNum())));
assertThat("Sync query reply contained wrong timestamp", timestamp, is(equalTo(queryResponse.getLastTimeOfSync())));
}
public void close(){
signers[0].clearSigningKey();

36
bulletin-board-server/src/test/java/meerkat/bulletinboard/H2BulletinBoardServerTest.java
View File

@ -7,7 +7,6 @@ import meerkat.comm.CommunicationException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.Result;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
@ -55,7 +54,7 @@ public class H2BulletinBoardServerTest {
BulletinBoardServer bulletinBoardServer = new BulletinBoardSQLServer(queryProvider);
try {
bulletinBoardServer.init("");
bulletinBoardServer.init();
} catch (CommunicationException e) {
System.err.println(e.getMessage());
@ -107,6 +106,39 @@ public class H2BulletinBoardServerTest {
System.err.println("Time of operation: " + (end - start));
}
@Test
public void testBatch() {
final int BATCH_NUM = 20;
try{
for (int i = 0 ; i < BATCH_NUM ; i++) {
serverTest.testPostBatch();
}
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
try{
serverTest.testReadBatch();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@Test
public void testSyncQuery() {
try {
serverTest.testSyncQuery();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@After
public void close() {
System.err.println("Starting to close H2BulletinBoardServerTest");

36
bulletin-board-server/src/test/java/meerkat/bulletinboard/MySQLBulletinBoardServerTest.java
View File

@ -11,7 +11,6 @@ import org.junit.Test;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
@ -59,7 +58,7 @@ public class MySQLBulletinBoardServerTest {
BulletinBoardServer bulletinBoardServer = new BulletinBoardSQLServer(queryProvider);
try {
bulletinBoardServer.init("");
bulletinBoardServer.init();
} catch (CommunicationException e) {
System.err.println(e.getMessage());
@ -111,6 +110,39 @@ public class MySQLBulletinBoardServerTest {
System.err.println("Time of operation: " + (end - start));
}
@Test
public void testBatch() {
final int BATCH_NUM = 20;
try{
for (int i = 0 ; i < BATCH_NUM ; i++) {
serverTest.testPostBatch();
}
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
try{
serverTest.testReadBatch();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@Test
public void testSyncQuery() {
try {
serverTest.testSyncQuery();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@After
public void close() {
System.err.println("Starting to close MySQLBulletinBoardServerTest");

27
bulletin-board-server/src/test/java/meerkat/bulletinboard/SQLiteBulletinBoardServerTest.java
View File

@ -39,7 +39,7 @@ public class SQLiteBulletinBoardServerTest{
BulletinBoardServer bulletinBoardServer = new BulletinBoardSQLServer(new SQLiteQueryProvider(testFilename));
try {
bulletinBoardServer.init("");
bulletinBoardServer.init();
} catch (CommunicationException e) {
System.err.println(e.getMessage());
@ -60,7 +60,7 @@ public class SQLiteBulletinBoardServerTest{
System.err.println("Time of operation: " + (end - start));
}
@Test
// @Test
public void bulkTest() {
System.err.println("Starting bulkTest of SQLiteBulletinBoardServerTest");
long start = threadBean.getCurrentThreadCpuTime();
@ -91,6 +91,29 @@ public class SQLiteBulletinBoardServerTest{
System.err.println("Time of operation: " + (end - start));
}
// @Test
public void testBatch() {
final int BATCH_NUM = 20;
try{
for (int i = 0 ; i < BATCH_NUM ; i++) {
serverTest.testPostBatch();
}
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
try{
serverTest.testReadBatch();
} catch (Exception e) {
System.err.println(e.getMessage());
fail(e.getMessage());
}
}
@After
public void close() {
System.err.println("Starting to close SQLiteBulletinBoardServerTest");

223
distributed-key-generation/build.gradle Normal file
View File

@ -0,0 +1,223 @@
plugins {
id "us.kirchmeier.capsule" version "1.0.1"
id 'com.google.protobuf' version '0.7.0'
}
apply plugin: 'java'
apply plugin: 'eclipse'
apply plugin: 'idea'
apply plugin: 'maven-publish'
// Uncomment the lines below to define an application
// (this will also allow you to build a "fatCapsule" which includes
// the entire application, including all dependencies in a single jar)
//apply plugin: 'application'
//mainClassName='your.main.ApplicationClass'
// Is this a snapshot version?
ext { isSnapshot = false }
ext {
groupId = 'org.factcenter.meerkat'
nexusRepository = "https://cs.idc.ac.il/nexus/content/groups/${isSnapshot ? 'unstable' : 'public'}/"
// Credentials for IDC nexus repositories (needed only for using unstable repositories and publishing)
// Should be set in ${HOME}/.gradle/gradle.properties
nexusUser = project.hasProperty('nexusUser') ? project.property('nexusUser') : ""
nexusPassword = project.hasProperty('nexusPassword') ? project.property('nexusPassword') : ""
}
description = "TODO: Add a description"
// Your project version
version = "0.0"
version += "${isSnapshot ? '-SNAPSHOT' : ''}"
dependencies {
// Meerkat common
compile project(':meerkat-common')
// Logging
compile 'org.slf4j:slf4j-api:1.7.7'
runtime 'ch.qos.logback:logback-classic:1.1.2'
runtime 'ch.qos.logback:logback-core:1.1.2'
// Google protobufs
compile 'com.google.protobuf:protobuf-java:3.+'
// Depend on test resources from meerkat-common
testCompile project(path: ':meerkat-common', configuration: 'testOutput')
testCompile 'junit:junit:4.+'
runtime 'org.codehaus.groovy:groovy:2.4.+'
}
/*==== You probably don't have to edit below this line =======*/
// Setup test configuration that can appear as a dependency in
// other subprojects
configurations {
testOutput.extendsFrom (testCompile)
}
task testJar(type: Jar, dependsOn: testClasses) {
classifier = 'tests'
from sourceSets.test.output
}
artifacts {
testOutput testJar
}
// The run task added by the application plugin
// is also of type JavaExec.
tasks.withType(JavaExec) {
// Assign all Java system properties from
// the command line to the JavaExec task.
systemProperties System.properties
}
protobuf {
// Configure the protoc executable
protoc {
// Download from repositories
artifact = 'com.google.protobuf:protoc:3.+'
}
}
idea {
module {
project.sourceSets.each { sourceSet ->
def srcDir = "${protobuf.generatedFilesBaseDir}/$sourceSet.name/java"
println "Adding $srcDir"
// add protobuf generated sources to generated source dir.
if ("test".equals(sourceSet.name)) {
testSourceDirs += file(srcDir)
} else {
sourceDirs += file(srcDir)
}
generatedSourceDirs += file(srcDir)
}
// Don't exclude build directory
excludeDirs -= file(buildDir)
}
}
/*===================================
* "Fat" Build targets
*===================================*/
if (project.hasProperty('mainClassName') && (mainClassName != null)) {
task mavenCapsule(type: MavenCapsule) {
description = "Generate a capsule jar that automatically downloads and caches dependencies when run."
applicationClass mainClassName
destinationDir = buildDir
}
task fatCapsule(type: FatCapsule) {
description = "Generate a single capsule jar containing everything. Use -Pfatmain=... to override main class"
destinationDir = buildDir
def fatMain = hasProperty('fatmain') ? fatmain : mainClassName
applicationClass fatMain
def testJar = hasProperty('test')
if (hasProperty('fatmain')) {
appendix = "fat-${fatMain}"
} else {
appendix = "fat"
}
if (testJar) {
from sourceSets.test.output
}
}
}
/*===================================
* Repositories
*===================================*/
repositories {
// Prefer the local nexus repository (it may have 3rd party artifacts not found in mavenCentral)
maven {
url nexusRepository
if (isSnapshot) {
credentials { username
password
username nexusUser
password nexusPassword
}
}
}
// Use local maven repository
mavenLocal()
// Use 'maven central' for other dependencies.
mavenCentral()
}
task "info" << {
println "Project: ${project.name}"
println "Description: ${project.description}"
println "--------------------------"
println "GroupId: $groupId"
println "Version: $version (${isSnapshot ? 'snapshot' : 'release'})"
println ""
}
info.description 'Print some information about project parameters'
/*===================================
* Publishing
*===================================*/
publishing {
publications {
mavenJava(MavenPublication) {
groupId project.groupId
pom.withXml {
asNode().appendNode('description', project.description)
}
from project.components.java
}
}
repositories {
maven {
url "https://cs.idc.ac.il/nexus/content/repositories/${project.isSnapshot ? 'snapshots' : 'releases'}"
credentials { username
password
username nexusUser
password nexusPassword
}
}
}
}

47
distributed-key-generation/src/main/java/meerkat/crypto/dkg/comm/MessageHandler.java Normal file
View File

@ -0,0 +1,47 @@
package meerkat.crypto.dkg.comm;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.comm.Channel;
import meerkat.protobuf.Comm;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Created by Tzlil on 2/14/2016.
*
* an implementation of ReceiverCallback
*/
public abstract class MessageHandler implements Channel.ReceiverCallback {
final Logger logger = LoggerFactory.getLogger(getClass());
/**
* fixed value for broadcasting
*/
public static final int BROADCAST = 0;
/**
* Handle a broadcast (or unicast) message.
* If the message is invalid, the handler can throw an {@link InvalidProtocolBufferException}, in which
* case the message will simply be ignored.
* @param envelope
*/
public abstract void handleMessage(Comm.BroadcastMessage envelope) throws InvalidProtocolBufferException;
/**
* Was this broadcastMessage was received by broadcast channel
* @param broadcastMessage
* @return broadcastMessage user destination == BROADCAST
*/
public boolean isBroadcast(Comm.BroadcastMessage broadcastMessage){
return broadcastMessage.getDestination() == BROADCAST;
}
@Override
public void receiveMessage(Comm.BroadcastMessage envelope) {
try {
handleMessage(envelope);
} catch (InvalidProtocolBufferException e) {
logger.warn("Received invalid protocol buffer from channel", e);
}
}
}

36
distributed-key-generation/src/main/java/meerkat/crypto/dkg/comm/MessageUtils.java Normal file
View File

@ -0,0 +1,36 @@
package meerkat.crypto.dkg.comm;
import meerkat.protobuf.DKG;
/**
* Created by talm on 12/04/16.
*/
public class MessageUtils {
public static DKG.Payload createMessage(DKG.Payload.Type type) {
return DKG.Payload.newBuilder().setType(type).build();
}
public static DKG.Payload createMessage(DKG.Payload.Type type, DKG.ShareMessage share) {
return DKG.Payload.newBuilder().setType(type).setShare(share).build();
}
public static DKG.Payload createMessage(DKG.Payload.Type type, DKG.ShareMessage.Builder share) {
return DKG.Payload.newBuilder().setType(type).setShare(share).build();
}
public static DKG.Payload createMessage(DKG.Payload.Type type, DKG.IDMessage id) {
return DKG.Payload.newBuilder().setType(type).setId(id).build();
}
public static DKG.Payload createMessage(DKG.Payload.Type type, DKG.IDMessage.Builder id) {
return DKG.Payload.newBuilder().setType(type).setId(id).build();
}
public static DKG.Payload createMessage(DKG.Payload.Type type, DKG.CommitmentMessage commitment) {
return DKG.Payload.newBuilder().setType(type).setCommitment(commitment).build();
}
public static DKG.Payload createMessage(DKG.Payload.Type type, DKG.CommitmentMessage.Builder commitment) {
return DKG.Payload.newBuilder().setType(type).setCommitment(commitment).build();
}
}

40
distributed-key-generation/src/main/java/meerkat/crypto/dkg/feldman/Party.java Normal file
View File

@ -0,0 +1,40 @@
package meerkat.crypto.dkg.feldman;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import java.util.ArrayList;
import java.util.Arrays;
/**
* Created by Tzlil on 3/14/2016.
*
* contains all relevant information on specific party during
* the run of Joint Feldamn protocol
*/
public class Party<T> {
public final int id;
public Polynomial.Point share;
public ArrayList<T> commitments;
public boolean doneFlag;
public Protocol.ComplaintState[] complaints;
public boolean aborted;
/**
*
* @param id party identifier - 1 <= id <= n
* @param n number of parties in current run protocol
* @param t protocol's threshold
*/
public Party(int id, int n, int t) {
this.id = id;
this.share = null;
this.doneFlag = false;
this.complaints = new Protocol.ComplaintState[n];
Arrays.fill(this.complaints, Protocol.ComplaintState.OK);
this.commitments = new ArrayList<T>(t + 1);
for (int i = 0; i <= t ; i++){
commitments.add(null);
}
this.aborted = false;
}
}

369
distributed-key-generation/src/main/java/meerkat/crypto/dkg/feldman/Protocol.java Normal file
View File

@ -0,0 +1,369 @@
package meerkat.crypto.dkg.feldman;
import meerkat.comm.Channel;
import meerkat.crypto.secretsharing.feldman.VerifiableSecretSharing;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import com.google.protobuf.ByteString;
import meerkat.protobuf.DKG;
import org.factcenter.qilin.primitives.Group;
import org.factcenter.qilin.util.ByteEncoder;
import java.math.BigInteger;
import java.util.*;
import static meerkat.crypto.dkg.comm.MessageUtils.*;
/**
* Created by Tzlil on 3/14/2016.
*
* an implementation of JointFeldman distributed key generation protocol.
*
* allows set of n parties to generate random key with threshold t.
*/
public class Protocol<T> extends VerifiableSecretSharing<T> {
public enum ComplaintState {
/**
* No complaints, no response required at this point.
*/
OK,
/**
* Party received complaint, waiting for response from party
*/
Waiting,
/**
* Party gave invalid answer to complaint.
*/
Disqualified,
/**
* Party received complaint, gave valid answer.
*/
NonDisqualified
}
/**
* My share id.
*/
protected final int id;
/**
* All parties participating in key generation.
* parties[id-1] has my info.
*/
private Party<T>[] parties;
/**
* communication object
*/
protected Channel channel;
/**
* Encode/Decode group elements
*/
protected final ByteEncoder<T> encoder;
/**
* constructor
* @param q a large prime.
* @param t threshold. Any t+1 share holders can recover the secret,
* but any set of at most t share holders cannot
* @param n number of share holders
* @param zi secret, chosen from Zq
* @param random use for generate random polynomial
* @param group
* @param q a large prime dividing group order.
* @param g a generator of cyclic group of order q.
* the generated group is a subgroup of the given group.
* it must be chosen such that computing discrete logarithms is hard in this group.
* @param encoder Encode/Decode group elements (of type T) to/from byte array
*/
//TODO: why the use of regular Random? Should it be changed?
public Protocol(int t, int n, BigInteger zi, Random random, BigInteger q, T g
, Group<T> group, int id, ByteEncoder<T> encoder) {
super(t, n, zi, random, q, g,group);
this.id = id;
this.parties = new Party[n];
for (int i = 1; i <= n ; i++){
this.parties[i - 1] = new Party(i,n,t);
}
this.parties[id - 1].share = getShare(id);
this.encoder = encoder;
}
/**
* setter
* @param channel
*/
public void setChannel(Channel channel){
this.channel = channel;
}
/**
* setter
* @param parties
*/
protected void setParties(Party[] parties){
this.parties = parties;
}
/**
* getter
* @return
*/
protected Party[] getParties(){
return parties;
}
/**
* stage1.1 according to the protocol
* Pi broadcasts Aik for k = 0,...,t.
*/
public void broadcastCommitments(){
broadcastCommitments(commitmentsArrayList);
}
/**
* pack commitments as messages and broadcast them
* @param commitments
*/
public void broadcastCommitments(ArrayList<T> commitments){
DKG.CommitmentMessage commitmentMessage;
for (int k = 0; k <= t ; k++){
commitmentMessage = DKG.CommitmentMessage.newBuilder()
.setCommitment(ByteString.copyFrom(encoder.encode(commitments.get(k))))
.setK(k)
.build();
channel.broadcastMessage(createMessage(DKG.Payload.Type.COMMITMENT, commitmentMessage));
}
}
/**
* Send channel j her secret share (of my polynomial)
* @param j
*/
public void sendSecret(int j){
ByteString secret = ByteString.copyFrom(getShare(j).y.toByteArray());
channel.sendMessage(j, createMessage(DKG.Payload.Type.SHARE,
DKG.ShareMessage.newBuilder()
.setI(id)
.setJ(j)
.setShare(secret)
));
}
/**
* stage1.2 according to the protocol
* Pi computes the shares Sij for j = 1,...,n and sends Sij secretly to Pj.
*/
public void sendSecrets(){
for (int j = 1; j <= n ; j++){
if(j != id){
sendSecret(j);
}
}
}
/**
*
* @param i
* @return computeVerificationValue(j,parties[i - 1].commitments,group) == g ^ parties[i - 1].share mod q
*/
public boolean isValidShare(int i){
Party<T> party = parties[i - 1];
synchronized (parties[i - 1]) {
return isValidShare(party.share, party.commitments, id);
}
}
/**
* @param share
* @param commitments
* @param j
* @return computeVerificationValue(j,commitments,group) == g ^ secret.y mod q
*/
public boolean isValidShare(Polynomial.Point share, ArrayList<T> commitments, int j){
try{
T v = computeVerificationValue(j,commitments,group);
return group.multiply(g,share.y).equals(v);
}
catch (NullPointerException e){
return false;
}
}
/**
* stage2 according to the protocol
* Pj verifies all the shares he received (using isValidShare)
* if check fails for an index i, Pj broadcasts a complaint against Pi.
*/
public void broadcastComplaints(){
for (int i = 1; i <= n ; i++ ){
if(i != id && !isValidShare(i)) {
broadcastComplaint(i);
}
}
}
/**
* create a complaint message against i and broadcast it
* @param i
*/
private void broadcastComplaint(int i){
//message = new Message(Type.Complaint, j)
DKG.IDMessage complaint = DKG.IDMessage.newBuilder()
.setId(i)
.build();
channel.broadcastMessage(createMessage(DKG.Payload.Type.COMPLAINT, complaint));
}
/**
* create an answer message for j and broadcast it
* @param j
*/
public void broadcastComplaintAnswer(int j){
channel.broadcastMessage(createMessage(DKG.Payload.Type.ANSWER, DKG.ShareMessage.newBuilder()
.setI(id)
.setJ(j)
.setShare(ByteString.copyFrom(getShare(j).y.toByteArray()))));
}
/**
* stage3.1 according to the protocol
* if more than t players complain against a player Pi he is disqualified.
*/
public void answerAllComplainingPlayers(){
ComplaintState[] complaints = parties[id - 1].complaints;
for (int i = 1; i <= n; i++) {
switch (complaints[i - 1]) {
case Waiting:
broadcastComplaintAnswer(i);
break;
default:
break;
}
}
}
/**
* stage3.2 according to the protocol
* if any of the revealed shares fails the verification test, player Pi is disqualified.
* set QUAL to be the set of non-disqualified players.
*/
public Set<Integer> calcQUAL(){
Set<Integer> QUAL = new HashSet<Integer>();
boolean nonDisqualified;
int counter;
for (int i = 1; i <= n; i++) {
synchronized (parties[i - 1]) {
ComplaintState[] complaints = parties[i - 1].complaints;
nonDisqualified = true;
counter = 0;
for (int j = 1; j <= n; j++) {
switch (complaints[j - 1]) {
case OK:
break;
case NonDisqualified:
counter++;
break;
default:
nonDisqualified = false;
break;
}
if (!nonDisqualified)
break;
}
if (nonDisqualified && counter <= t) {
QUAL.add(i);
}
}
}
return QUAL;
}
/**
* compute Y, the commitment to the final public key (includes only qualifying set)
* stage4.1 according to the protocol
* public value y is computed as y = multiplication of yi mod p for i in QUAL
*/
public T calcY(Set<Integer> QUAL){
T y = group.zero();
for (int i : QUAL) {
synchronized (parties[i - 1]) {
y = group.add(y, parties[i - 1].commitments.get(0));
}
}
return y;
}
/**
* stage4.2 according to the protocol
* public verification values are computed as Ak = multiplication
* of Aik mod p for i in QUAL for k = 0,...,t
*/
public ArrayList<T> calcCommitments(Set<Integer> QUAL){
ArrayList<T> commitments = new ArrayList<T>(t+1);
T value;
for (int k = 0; k <= t; k++){
value = group.zero();
for (int i : QUAL) {
synchronized (parties[i - 1]) {
value = group.add(value, parties[i - 1].commitments.get(k));
}
}
commitments.add(k,value);
}
return commitments;
}
/**
* stage4.3 according to the protocol
* Pj sets is share of the share as xj = sum of Sij mod q for i in QUAL
*/
public Polynomial.Point calcShare(Set<Integer> QUAL){
BigInteger xj = BigInteger.ZERO;
for (int i : QUAL) {
synchronized (parties[i - 1]) {
xj = xj.add(parties[i - 1].share.y);
}
}
return new Polynomial.Point(BigInteger.valueOf(id) , xj.mod(q));
}
/**
* decode commitment from arr
* @param arr
* @return
*/
public T decodeCommitment(byte[] arr){
return encoder.decode(arr);
}
/**
* getter
* @return id
*/
public int getId() {
return id;
}
/**
* getter
* @return channel
*/
public Channel getChannel() {
return channel;
}
/**
* getter
* @return encoder
*/
public ByteEncoder<T> getEncoder() {
return encoder;
}
}

588
distributed-key-generation/src/main/java/meerkat/crypto/dkg/feldman/User.java Normal file
View File

@ -0,0 +1,588 @@
package meerkat.crypto.dkg.feldman;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.TextFormat;
import meerkat.comm.Channel;
import meerkat.crypto.dkg.comm.MessageHandler;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import com.google.protobuf.ByteString;
import meerkat.protobuf.Comm;
import meerkat.protobuf.DKG;
import org.factcenter.qilin.primitives.Group;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;
import static meerkat.crypto.dkg.comm.MessageUtils.createMessage;
/**
* Created by Tzlil on 3/14/2016.
*
* implementation of joint feldman protocol user.
*
* according to the protocol, each user run feldman verifiable secret sharing
* as a dealer.
*
* by the end of run(), each party in QUAL has his own share of the generated random key.
* this key can be recover by any subset of QUAL of size at least t + 1.
*/
public class User<T> implements Runnable {
final Logger logger = LoggerFactory.getLogger(getClass());
/**
* joint feldman protocol object
*/
protected final Protocol<T> dkg;
/**
* a generator of cyclic group of order q.
* the generated group is a subgroup of the given group.
* it must be chosen such that computing discrete logarithms is hard in this group.
*/
protected final T g;
/**
* cyclic group contains g.
*/
protected final Group<T> group;
/**
* user id
*/
protected final int id;
/**
* threshold
*/
protected final int t;
/**
* number of shares
*/
protected final int n;
/**
* channel object
*/
protected final Channel channel; //
/**
* All parties participating in key generation.
* parties[id-1] has my info.
*/
protected final Party[] parties;
/**
* set of all non-disqualified parties
*/
protected Set<Integer> QUAL;
/**
* my own share of the generated random key.
*/
protected Polynomial.Point share;
/**
* public verification values
*/
protected ArrayList<T> commitments;
/**
* public value,
* y = g ^ key
*/
protected T y;
protected BlockingQueue<Comm.BroadcastMessage> receiveQueue;
/**
* constructor
* @param dkg joint feldman protocol object
* @param channel channel object
*/
public User(Protocol<T> dkg, Channel channel) {
this.dkg = dkg;
this.g = dkg.getGenerator();
this.group = dkg.getGroup();
this.n = dkg.getN();
this.t = dkg.getT();
this.id = dkg.getId();
this.channel = channel;
dkg.setChannel(channel);
registerReceiverCallback();
this.parties = dkg.getParties();
this.QUAL = null;
this.commitments = null;
this.share = null;
this.y = null;
this.receiveQueue = new LinkedBlockingDeque<>();
}
/**
* create MessageHandler and register it as ReceiverCallback
*/
protected void registerReceiverCallback() {
channel.registerReceiverCallback(new meerkat.crypto.dkg.comm.MessageHandler() {
@Override
public void handleMessage(Comm.BroadcastMessage envelope) throws InvalidProtocolBufferException {
receiveQueue.add(envelope);
}
});
// this.messageHandler = new MessageHandler();
// channel.registerReceiverCallback(messageHandler);
}
/**
* Wait for at least one message to arrive, then handle any messages currently in the queue
*/
protected void waitAndHandleReceivedMessages() {
Comm.BroadcastMessage msg = null;
while (!stop && msg == null) {
try {
msg = receiveQueue.take();
} catch (InterruptedException e) {
// Possibly stop
}
}
while (!stop && msg != null) {
try {
handleMessage(msg);
} catch (InvalidProtocolBufferException e) {
logger.warn("Received invalid message: {}", TextFormat.printToString(msg));
}
msg = receiveQueue.poll();
}
}
/**
* stage1 according to the protocol
* 1. Pi broadcasts Aik for k = 0,...,t.
* 2. Pi computes the shares Sij for j = 1,...,n and sends Sij secretly to Pj.
*/
protected void stage1() {
dkg.broadcastCommitments();
dkg.sendSecrets();
}
/**
* Check if all shares and commitments have arrived from other parties
*/
protected boolean isStageOneCompleted() {
for (int i = 0 ; i < n ; i++) {
if (!parties[i].aborted) {
if (parties[i].share == null)
return false;
for (int k = 0 ; k <= t ; k++) {
if (parties[i].commitments.get(k) == null)
return false;
}
}
}
return true;
}
protected void waitUntilStageOneCompleted() {
while (!stop && !isStageOneCompleted())
waitAndHandleReceivedMessages();
}
protected boolean isStageTwoCompleted() {
for (int i = 0 ; i < n ; i++) {
if (!parties[i].aborted && !parties[i].doneFlag)
return false;
}
return true;
}
/**
* stage2 according to the protocol
* Pj verifies all the shares he received
* if check fails for an index i, Pj broadcasts a complaint against Pi.
* Pj broadcasts done message at the end of this stage
*/
protected void stage2() {
dkg.broadcastComplaints();
//broadcast done message after all complaints
channel.broadcastMessage(createMessage(DKG.Payload.Type.DONE));
}
/**
* wait until all other parties done complaining by receiving done message
*/
protected void waitUntilStageTwoCompleted(){
while (!stop && !isStageTwoCompleted())
waitAndHandleReceivedMessages();
}
protected boolean haveReceivedAllStage3ComplaintAnswers() {
for (int i = 0; i < n; i++) {
if (parties[i].aborted)
continue;
for (int j = 0; j < n; j++) {
if (parties[i].complaints[j].equals(Protocol.ComplaintState.Waiting))
return false;
}
}
return true;
}
/**
* stage3 according to the protocol
* 1. if more than t players complain against a player Pi he is disqualified.
* otherwise Pi broadcasts the share Sij for each complaining player Pj.
* 2. if any of the revealed shares fails the verification test, player Pi is disqualified.
* set QUAL to be the set of non-disqualified players.
*/
protected void stage3(){
dkg.answerAllComplainingPlayers();
// wait until there is no complaint waiting for answer
while (!stop && !haveReceivedAllStage3ComplaintAnswers())
waitAndHandleReceivedMessages();
this.QUAL = dkg.calcQUAL();
}
/**
* stage4 according to the protocol
* 1. public value y is computed as y = multiplication of yi mod p for i in QUAL
* 2. public verification values are computed as Ak = multiplication of Aik mod p for i in QUAL for k = 0,...,t
* 3. Pj sets is share of the secret as xj = sum of Sij mod q for i in QUAL
*/
protected void stage4(){
this.y = dkg.calcY(QUAL);
this.commitments = dkg.calcCommitments(QUAL);
this.share = dkg.calcShare(QUAL);
}
@Override
public void run() {
this.runThread = Thread.currentThread();
// For debugging
String previousName = runThread.getName();
runThread.setName(getClass().getName() +":" + getID());
try {
stage1();
waitUntilStageOneCompleted();
if (stop) return;
stage2();
waitUntilStageTwoCompleted();
if (stop) return;
stage3();
if (stop) return;
stage4();
} finally {
runThread.setName(previousName);
}
}
/**
* current thread in the main loop
*/
protected Thread runThread;
/**
* flag indicates if there was request to stop the current run of the protocol
*/
protected boolean stop = false;
/**
* Request the current run loop to exit gracefully
*/
public void stop() {
try {
stop = true;
runThread.interrupt();
}catch (Exception e){
//do nothing
}
}
/**
* getter
* @return commitments
*/
public ArrayList<T> getCommitments() {
return commitments;
}
/**
* getter
* @return g
*/
public T getGenerator() {
return g;
}
/**
* getter
* @return group
*/
public Group<T> getGroup() {
return group;
}
/**
* getter
* @return share
*/
public Polynomial.Point getShare() {
return share;
}
/**
* getter
* @return id
*/
public int getID() {
return id;
}
/**
* getter
* @return n
*/
public int getN() {
return n;
}
/**
* getter
* @return t
*/
public int getT() {
return t;
}
/**
* getter
* @return y
*/
public T getPublicValue() {
return y;
}
/**
* getter
* @return QUAL
*/
public Set<Integer> getQUAL() {
return QUAL;
}
/**
* getter
* @return channel
*/
public Channel getChannel() {
return channel;
}
/**
* commitment message is valid if:
* 1. it was received in broadcast chanel
* 2. the sender didn't sent this commitment before
*/
protected boolean isValidCommitmentMessage(int sender, boolean isBroadcast, DKG.CommitmentMessage commitmentMessage){
int i = sender - 1;
int k = commitmentMessage.getK();
return isBroadcast && parties[i].commitments.get(k) == null;
}
/**
* secret message is valid if:
* 1. it was received in private chanel
* 2. the sender didn't sent secret message before
* 3. secret.i == i
* 4. secret.j == id
*/
protected boolean isValidSecretMessage(int sender, boolean isBroadcast, DKG.ShareMessage secretMessage){
int i = secretMessage.getI();
int j = secretMessage.getJ();
if(sender != i || isBroadcast)
return false;
else
return parties[i - 1].share == null && j == id;
}
/**
* done message is valid if:
* 1. it was received in broadcast chanel
* 2. the sender didn't sent done message before
*/
protected boolean isValidDoneMessage(int sender, boolean isBroadcast){
return isBroadcast && !parties[sender - 1].doneFlag;
}
/**
* complaint message is valid if:
* 1. it was received in broadcast chanel
* 2. the sender didn't complain against id before
*/
protected boolean isValidComplaintMessage(int sender, boolean isBroadcast, DKG.IDMessage complaintMessage){
int i = sender;
int j = complaintMessage.getId();
assert(i > 0);
assert(j > 0);
assert(i <= parties.length);
assert(j <= parties[i-1].complaints.length);
return isBroadcast && parties[i - 1].complaints[j - 1].equals( Protocol.ComplaintState.OK);
}
/**
* answer message is valid if:
* 1. it was received in broadcast chanel
* 2. secret.i == i
* 3. 1 <= secret.j <= n
* 4. it is marked that j complained against i and i didn't received
*/
protected boolean isValidAnswerMessage(int sender, boolean isBroadcast, DKG.ShareMessage secretMessage){
int i = secretMessage.getI();
int j = secretMessage.getJ();
if(sender != i || !isBroadcast)
return false;
else
return j >= 1 && j <= n && parties[i - 1].complaints[j - 1].equals(Protocol.ComplaintState.Waiting);
}
public void handleMessage(Comm.BroadcastMessage envelope) throws InvalidProtocolBufferException {
int sender = envelope.getSender();
boolean isBroadcast = !envelope.getIsPrivate();
DKG.Payload msg = DKG.Payload.parseFrom(envelope.getPayload());
logger.debug("handling Message: Dst={}, Src={}, [{}]",
envelope.getDestination(), envelope.getSender(), TextFormat.printToString(msg));
switch (msg.getType()) {
case COMMITMENT:
/**
* saves the commitment
*/
assert msg.getPayloadDataCase() == DKG.Payload.PayloadDataCase.COMMITMENT;
DKG.CommitmentMessage commitmentMessage = msg.getCommitment();
if (isValidCommitmentMessage(sender, isBroadcast, commitmentMessage)) {
int i = sender - 1;
int k = commitmentMessage.getK();
parties[i].commitments.set(k, extractCommitment(commitmentMessage));
}
break;
case SHARE:
/**
* saves the secret
*/
assert msg.getPayloadDataCase() == DKG.Payload.PayloadDataCase.SHARE;
DKG.ShareMessage secretMessage = msg.getShare();
if(isValidSecretMessage(sender,isBroadcast,secretMessage)) {
int i = secretMessage.getI();
Polynomial.Point secret = extractShare(id,secretMessage.getShare());
parties[i - 1].share = secret;
}
break;
case DONE:
/**
* marks that the sender was finished sending all his complaints
*/
if(isValidDoneMessage(sender,isBroadcast)) {
parties[sender - 1].doneFlag = true;
}
break;
case COMPLAINT:
/**
* marks that the sender was complained against id
*/
if (msg.getPayloadDataCase() != DKG.Payload.PayloadDataCase.ID) {
logger.error("User {} Expecting ID message, got from SRC={} msg {}", getID(), envelope.getSender(), TextFormat.printToString(msg));
assert (msg.getPayloadDataCase() == DKG.Payload.PayloadDataCase.ID);
}
DKG.IDMessage complaintMessage = msg.getId();
if(isValidComplaintMessage(sender,isBroadcast,complaintMessage)){
int i = sender;
int j = complaintMessage.getId();
parties[j - 1].complaints[i - 1] = Protocol.ComplaintState.Waiting;
}
break;
case ANSWER:
/**
* if the secret is valid, marks the complaint as NonDisqualified
* else marks it as Disqualified
* in case that the complainer is id ( j == id ), saves the secret
*/
assert msg.getPayloadDataCase() == DKG.Payload.PayloadDataCase.SHARE;
secretMessage = msg.getShare();
if(isValidAnswerMessage(sender,isBroadcast,secretMessage)) {
int i = secretMessage.getI();
int j = secretMessage.getJ();
Polynomial.Point secret = extractShare(j,secretMessage.getShare());
if (dkg.isValidShare(secret, parties[i - 1].commitments, j)) {
parties[i - 1].complaints[j - 1] = Protocol.ComplaintState.NonDisqualified;
} else {
parties[i - 1].complaints[j - 1] = Protocol.ComplaintState.Disqualified;
}
if (j == id) {
parties[i - 1].share = secret;
}
}
break;
case ABORT:
/**
* marks that the sender was aborted
*/
parties[sender - 1].aborted = true;
break;
default:
logger.error("Bad message: SRC={}, DST={}, Payload={}", envelope.getSender(), envelope.getDestination(), TextFormat.printToString(msg));
break;
}
}
/**
* extract share value from ByteString
* @param i
* @param share
* @return new Point (i,share)
*/
public Polynomial.Point extractShare(int i, ByteString share){
BigInteger x = BigInteger.valueOf(i);
BigInteger y = new BigInteger(share.toByteArray());
return new Polynomial.Point(x,y);
}
/**
*
* @param commitmentMessage
* @return
*/
public T extractCommitment(DKG.CommitmentMessage commitmentMessage){
return dkg.decodeCommitment(commitmentMessage.getCommitment().toByteArray());
}
}

28
distributed-key-generation/src/main/java/meerkat/crypto/dkg/gjkr/Party.java Normal file
View File

@ -0,0 +1,28 @@
package meerkat.crypto.dkg.gjkr;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Set;
/**
* Created by Tzlil on 3/16/2016.
*
* an extension of DistributedKeyGenerationParty
* contains all relevant information on specific party during
* the run of the safe protocol
*/
public class Party<T> extends meerkat.crypto.dkg.feldman.Party<T> {
public Polynomial.Point shareT;
public boolean ysDoneFlag;
public ArrayList<T> verifiableValues;
public Set<Polynomial.Point> recoverSharesSet;
public Party(int id, int n, int t) {
super(id, n, t);
this.shareT = null;
this.ysDoneFlag = false;
this.verifiableValues = new ArrayList<T>(this.commitments);
this.recoverSharesSet = new HashSet<Polynomial.Point>();
}
}

165
distributed-key-generation/src/main/java/meerkat/crypto/dkg/gjkr/Protocol.java Normal file
View File

@ -0,0 +1,165 @@
package meerkat.crypto.dkg.gjkr;
import meerkat.crypto.dkg.comm.MessageUtils;
import meerkat.crypto.secretsharing.feldman.VerifiableSecretSharing;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import com.google.protobuf.ByteString;
import meerkat.protobuf.DKG;
import org.factcenter.qilin.primitives.Group;
import org.factcenter.qilin.util.ByteEncoder;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Random;
import java.util.Set;
/**
* Created by Tzlil on 3/16/2016.
* TODO: comments
* TODO: put Channel (ChannelImpl) in constructor
*/
public class Protocol<T> extends meerkat.crypto.dkg.feldman.Protocol<T> {
private VerifiableSecretSharing<T> maskingShares;
private final T h;
private Party<T>[] parties;
public Protocol(int t, int n, BigInteger zi, Random random, BigInteger q, T g
, T h, Group<T> group, int id, ByteEncoder<T> byteEncoder) {
super(t, n, zi, random, q, g, group, id,byteEncoder);
this.h = h;
BigInteger r = new BigInteger(q.bitLength(),random).mod(q);
this.maskingShares = new VerifiableSecretSharing(t,n,r,random,q,h,group);
this.parties = new Party[n];
for (int i = 1; i <= n ; i++){
this.parties[i - 1] = new Party(i,n,t);
}
this.parties[id - 1].share = getShare(id);
this.parties[id - 1].shareT = maskingShares.getShare(id);
super.setParties(parties);
}
protected Party[] getParties(){
return parties;
}
protected void setParties(Party[] parties) {
super.setParties(parties);
this.parties = parties;
}
@Override
public void sendSecret(int j) {
Polynomial.Point secret = getShare(j);
Polynomial.Point secretT = maskingShares.getShare(j);
DKG.ShareMessage doubleSecretMessage = createShareMessage(id,j,secret,secretT);
// TODO: Change SHARE to SHARE
channel.sendMessage(j, MessageUtils.createMessage(DKG.Payload.Type.SHARE, doubleSecretMessage));
}
@Override
public boolean isValidShare(int i){
Party party = parties[i - 1];
return isValidShare(party.share, party.shareT, party.verifiableValues, id);
}
/**
* test if share, shareT are valid with respect to verificationValues
* @param share
* @param shareT
* @param verificationValues
* @param j
* @return computeVerificationValue(j,verificationValues,group) == (g ^ share.y) * (h ^ shareT.y) mod q
*/
public boolean isValidShare(Polynomial.Point share, Polynomial.Point shareT, ArrayList<T> verificationValues, int j){
try {
T v = computeVerificationValue(j, verificationValues, group);
T exp = group.add(group.multiply(g, share.y), group.multiply(h, shareT.y));
return exp.equals(v);
}
catch (NullPointerException e){
return false;
}
}
/**
* create complaint message against i and broadcast it
* @param share
* @param shareT
* @param i
*/
private void broadcastComplaint(Polynomial.Point share, Polynomial.Point shareT, int i){
DKG.ShareMessage complaint = createShareMessage(i,id,share,shareT);
channel.broadcastMessage(MessageUtils.createMessage(DKG.Payload.Type.COMPLAINT, complaint));
}
/**
* stage4.3 according to the protocol
* if check fails for index i, Pj
*/
public void computeAndBroadcastComplaints(Set<Integer> QUAL){
Party party;
for (int i : QUAL) {
party = parties[i - 1];
if (i != id) {
if (!super.isValidShare(party.share, party.commitments, id)) {
broadcastComplaint(party.share, party.shareT, i);
}
}
}
}
/**
* compute verification values and broadcast them
* verificationValues[k] = g ^ commitments [k] * h ^ maskingShares.commitments [k]
*/
public void computeAndBroadcastVerificationValues(){
ArrayList<T> verificationValues = new ArrayList<T>(t+1);
ArrayList<T> hBaseCommitments = maskingShares.getCommitmentsArrayList();
for (int k = 0 ; k <= t ; k++){
verificationValues.add(k,group.add(commitmentsArrayList.get(k),hBaseCommitments.get(k)));
}
broadcastCommitments(verificationValues);
}
/**
* pack share, shareT i,j to createShareMessage
* @param i
* @param j
* @param share
* @param shareT
* @return
*/
private DKG.ShareMessage createShareMessage(int i, int j, Polynomial.Point share, Polynomial.Point shareT){
DKG.ShareMessage ShareMessage = DKG.ShareMessage.newBuilder()
.setI(i)
.setJ(j)
.setShare(ByteString.copyFrom(share.y.toByteArray()))
.setShareT(ByteString.copyFrom(shareT.y.toByteArray()))
.build();
return ShareMessage;
}
@Override
public void broadcastComplaintAnswer(int j) {
DKG.ShareMessage answer = createShareMessage(id,j,getShare(j)
, maskingShares.getShare(j));
channel.broadcastMessage(MessageUtils.createMessage(DKG.Payload.Type.ANSWER, answer));
}
public void broadcastAnswer(Polynomial.Point secret, Polynomial.Point secretT, int i){
DKG.ShareMessage complaint = createShareMessage(i,id,secret,secretT);
channel.broadcastMessage(MessageUtils.createMessage(DKG.Payload.Type.ANSWER,complaint));
}
/**
* getter
* @return h
*/
public T getH() {
return h;
}
}

359
distributed-key-generation/src/main/java/meerkat/crypto/dkg/gjkr/User.java Normal file
View File

@ -0,0 +1,359 @@
package meerkat.crypto.dkg.gjkr;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.crypto.utils.Arithmetic;
import meerkat.crypto.utils.concrete.Fp;
import meerkat.comm.Channel;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import meerkat.crypto.secretsharing.shamir.SecretSharing;
import meerkat.protobuf.Comm;
import meerkat.protobuf.DKG;
import java.math.BigInteger;
import java.util.ArrayList;
import static meerkat.crypto.dkg.comm.MessageUtils.*;
/**
* Created by Tzlil on 3/16/2016.
* <p/>
* implementation of gjkr protocol user.
* <p/>
* this protocol extends joint Feldman protocol by splitting the protocol to commitment stage (stages 1,2,3)
* and revealing stage (stage 4).
* <p/>
* as in joint Feldman, each party in QUAL has his own share of the generated random key.
* this key can be recover by any subset of QUAL of size at least t + 1.
*/
public class User<T> extends meerkat.crypto.dkg.feldman.User<T> {
/**
* All parties participating in key generation.
* parties[id-1] has my info.
*/
protected Party<T>[] parties;
/**
* gjkr secure protocol object
*/
protected final Protocol<T> sdkg;
boolean isStage4;
/**
* constructor
*
* @param sdkg gjkr protocol object
* @param channel channel object
*/
public User(Protocol<T> sdkg, Channel channel) {
super(sdkg, channel);
this.sdkg = sdkg;
this.parties = sdkg.getParties();
}
/**
* stage1 according to the protocol
* 1. Pi broadcasts Cik=Aik*Bik for k = 0,...,t.
* 2. Pi computes the shares Sij,Sij' for j = 1,...,n and sends Sij,Sij' secretly to Pj.
*/
@Override
protected void stage1() {
sdkg.computeAndBroadcastVerificationValues();
sdkg.sendSecrets();
}
@Override
protected void waitUntilStageOneCompleted() {
super.waitUntilStageOneCompleted();
// save the received commitments as verification values
ArrayList<T> temp;
for (int i = 0; i < n; i++) {
temp = parties[i].verifiableValues;
parties[i].verifiableValues = parties[i].commitments;
parties[i].commitments = temp;
}
}
/**
* stage2 according to the protocol
* Pj verifies all the shares,sharesT he received
* if check fails for an index i, Pj broadcasts a complaint against Pi.
* Pj broadcasts done message at the end of this stage
*/
@Override
protected void stage2() {
sdkg.broadcastComplaints();
//broadcast done message after all complaints
channel.broadcastMessage(createMessage(DKG.Payload.Type.DONE));
}
/**
* Check if all non-aborting qualified parties have sent commitments.
* @return
*/
protected boolean haveAllQualPartiesCommitted() {
for (int i : QUAL) {
if (parties[i - 1].aborted)
continue;
for (int k = 0; k <= t; k++) {
if (parties[i - 1].commitments.get(k) == null)
return false;
}
}
return true;
}
/**
* Check if all non-aborting qualified parties sent a done message
* @return
*/
protected boolean areAllQualPartiesDone() {
for (int i : QUAL) {
if (parties[i - 1].aborted)
continue;
for (int k = 0; k <= t; k++) {
if (!parties[i - 1].ysDoneFlag)
return false;
}
}
return true;
}
/**
* Check if at least t + 1 secrets were received foreach i in QUAL that aborted
* @return
*/
protected boolean haveReceivedEnoughSecretShares() {
for (int i : QUAL) {
if (parties[i - 1].aborted && parties[i - 1].recoverSharesSet.size() <= t)
return false;
}
return true;
}
/**
* broadcast commitments and recover parties information if necessary
*/
private void resolveQualifyingPublicKey() {
sdkg.broadcastCommitments();
// wait until all parties in QUAL broadcast their commitments or aborted
while (!stop && !haveAllQualPartiesCommitted())
waitAndHandleReceivedMessages();
if (stop)
return;
sdkg.computeAndBroadcastComplaints(QUAL);
//broadcast done message after all complaints
channel.broadcastMessage(createMessage(DKG.Payload.Type.DONE));
// wait until all parties in QUAL done or aborted
while (!stop && !areAllQualPartiesDone())
waitAndHandleReceivedMessages();
if (stop)
return;
// broadcast i private secret foreach i in QUAL that aborted
for (int i : QUAL) {
if (parties[i - 1].aborted) {
sdkg.broadcastAnswer(parties[i - 1].share, parties[i - 1].shareT, i);
}
}
// wait until at least t + 1 secrets will received foreach i in QUAL that aborted
while (!stop && !haveReceivedEnoughSecretShares())
waitAndHandleReceivedMessages();
if (stop)
return;
Arithmetic<BigInteger> arithmetic = new Fp(sdkg.getQ());
// restore necessary information
for (int i = 0; i < n; i++) {
if (parties[i].recoverSharesSet.isEmpty()) {
continue;
}
Polynomial.Point[] shares = new Polynomial.Point[t + 1];
int j = 0;
for (Polynomial.Point share : parties[i].recoverSharesSet) {
shares[j++] = share;
if (j >= shares.length) {
break;
}
}
Polynomial polynomial = SecretSharing.recoverPolynomial(shares, arithmetic);
BigInteger[] coefficients = polynomial.getCoefficients();
for (int k = 0; k <= t; k++) {
parties[i].commitments.add(k, group.multiply(g, coefficients[k]));
}
parties[i].share = new Polynomial.Point(BigInteger.valueOf(id), polynomial);
}
}
/**
* notifies message handler and message handler that stage 4 was started
*/
protected void setStage4() {
isStage4 = true;
}
@Override
protected void stage4() {
setStage4();
resolveQualifyingPublicKey();
if (stop) return;
super.stage4();
}
/**
* if !isStage4 as super, with extension to double secret message
* else answer message is valid if:
* 1. it was received in broadcast chanel
* 2. secret.j == sender
* 3. QUAL contains i and j
*/
protected boolean isValidAnswerMessage(int sender, boolean isBroadcast, DKG.ShareMessage doubleSecretMessage) {
if (!isStage4) {
return super.isValidAnswerMessage(sender, isBroadcast, doubleSecretMessage);
} else {
int i = doubleSecretMessage.getI();
int j = doubleSecretMessage.getJ();
return isBroadcast && j == sender && parties[i - 1].aborted && !parties[j - 1].aborted
&& QUAL.contains(i) && QUAL.contains(j);
}
}
/**
* as in super with respect to protocol stage
*/
@Override
protected boolean isValidDoneMessage(int sender, boolean isBroadcast) {
if (!isStage4) {
return super.isValidDoneMessage(sender, isBroadcast);
} else {
return isBroadcast && !parties[sender - 1].ysDoneFlag;
}
}
/**
* use only in stage4
* complaint message is valid if:
* 1. it was received in broadcast chanel
* 2. secret.j == sender
* 3. QUAL contains i and j
*/
protected boolean isValidComplaintMessage(int sender, boolean isBroadcast,
DKG.ShareMessage complaintMessage) {
int i = complaintMessage.getI();
int j = complaintMessage.getJ();
return isBroadcast && j == sender && QUAL.contains(i) && QUAL.contains(j);
}
@Override
public void handleMessage(Comm.BroadcastMessage envelope) throws InvalidProtocolBufferException {
int sender = envelope.getSender();
boolean isBroadcast = !envelope.getIsPrivate();
DKG.Payload msg = DKG.Payload.parseFrom(envelope.getPayload());
switch (msg.getType()) {
case SHARE:
/**
* as in super, with extension to double secret message
*/
DKG.ShareMessage doubleSecretMessage = msg.getShare();
if (isValidSecretMessage(sender, isBroadcast, doubleSecretMessage)) {
int i = doubleSecretMessage.getI();
synchronized (parties[i - 1]) {
parties[i - 1].share = extractShare(id, doubleSecretMessage.getShare());
parties[i - 1].shareT = extractShare(id, doubleSecretMessage.getShareT());
parties[i - 1].notify();
}
}
break;
case ANSWER:
/**
* if !isStage4 as super, with extension to double secret message
* else saves secret
*/
assert msg.getPayloadDataCase() == DKG.Payload.PayloadDataCase.SHARE;
doubleSecretMessage = msg.getShare();
if (isValidAnswerMessage(sender, isBroadcast, doubleSecretMessage)) {
int i = doubleSecretMessage.getI();
int j = doubleSecretMessage.getJ();
Polynomial.Point secret = extractShare(j, doubleSecretMessage.getShare());
Polynomial.Point secretT = extractShare(j, doubleSecretMessage.getShareT());
synchronized (parties[i - 1]) {
if (!isStage4) {
if (sdkg.isValidShare(secret, secretT, parties[j - 1].verifiableValues, i)) {
parties[i - 1].complaints[j - 1] = meerkat.crypto.dkg.feldman.Protocol.ComplaintState.NonDisqualified;
} else {
parties[i - 1].complaints[j - 1] = meerkat.crypto.dkg.feldman.Protocol.ComplaintState.Disqualified;
}
if (j == id) {
parties[i - 1].share = secret;
parties[i - 1].shareT = secretT;
}
} else if (sdkg.isValidShare(secret, secretT, parties[i - 1].verifiableValues, j)) {
parties[i - 1].recoverSharesSet.add(secret);
}
parties[i - 1].notify();
}
}
break;
case DONE:
/**
* as in super with respect to protocol state
*/
if (!isStage4)
super.handleMessage(envelope);
else {
if (isValidDoneMessage(sender, isBroadcast)) {
synchronized (parties[sender - 1]) {
parties[sender - 1].ysDoneFlag = true;
parties[sender - 1].notify();
}
}
}
break;
case COMPLAINT:
/**
* if !isStage4 as in super
* else if secret,secretT are valid with respect to verifiableValues but
* secret is not valid with respect to commitments then
* marks i as aborted
*/
if (!isStage4) {
super.handleMessage(envelope);
} else {
assert (msg.getPayloadDataCase() == DKG.Payload.PayloadDataCase.SHARE);
DKG.ShareMessage ysComplaintMessage = msg.getShare();
if (isValidComplaintMessage(sender, isBroadcast, ysComplaintMessage)) {
int i = ysComplaintMessage.getI();
int j = ysComplaintMessage.getJ();
Polynomial.Point secret = extractShare(i, ysComplaintMessage.getShare());
Polynomial.Point secretT = extractShare(i, ysComplaintMessage.getShareT());
if (sdkg.isValidShare(secret, secretT, parties[i - 1].verifiableValues, j)
&& !dkg.isValidShare(secret, parties[i - 1].commitments, j)) {
synchronized (parties[i - 1]) {
parties[i - 1].aborted = true;
parties[i - 1].notify();
}
}
}
}
break;
default:
super.handleMessage(envelope);
break;
}
}
}

117
distributed-key-generation/src/main/java/meerkat/crypto/secretsharing/feldman/VerifiableSecretSharing.java Normal file
View File

@ -0,0 +1,117 @@
package meerkat.crypto.secretsharing.feldman;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import meerkat.crypto.secretsharing.shamir.SecretSharing;
import org.factcenter.qilin.primitives.Group;
import java.util.ArrayList;
import java.math.BigInteger;
import java.util.Random;
/**
* Created by Tzlil on 1/27/2016.
*
* an implementation of Feldman's verifiable secret sharing scheme.
*
* allows trusted dealer to share a key x among n parties.
*
*/
public class VerifiableSecretSharing<T> extends SecretSharing {
/**
* cyclic group contains g.
*/
protected final Group<T> group;
/**
* a generator of cyclic group of order q.
* the generated group is a subgroup of the given group.
* it must be chosen such that computing discrete logarithms is hard in this group.
*/
protected final T g;
/**
* commitments to polynomial coefficients.
* commitments[k] = g ^ coefficients[k] (group operation)
*/
protected final ArrayList<T> commitmentsArrayList;
/**
* constructor
* @param q a large prime.
* @param t threshold. Any t+1 share holders can recover the secret,
* but any set of at most t share holders cannot
* @param n number of share holders
* @param zi secret, chosen from Zq
* @param random use for generate random polynomial
* @param group
* @param q a large prime dividing group order.
* @param g a generator of cyclic group of order q.
* the generated group is a subgroup of the given group.
* it must be chosen such that computing discrete logarithms is hard in this group.
*/
public VerifiableSecretSharing(int t, int n, BigInteger zi, Random random, BigInteger q, T g
, Group<T> group) {
super(t, n, zi, random,q);
this.g = g;
this.group = group;
assert (this.group.contains(g));
this.commitmentsArrayList = generateCommitments();
}
/**
* commitments[i] = g ^ polynomial.coefficients[i]
* @return commitments
*/
private ArrayList<T> generateCommitments() {
Polynomial polynomial = getPolynomial();
BigInteger[] coefficients = polynomial.getCoefficients();
ArrayList<T> commitments = new ArrayList<T>(t + 1);
for (int i = 0 ; i <= t;i++){
commitments.add(i,group.multiply(g,coefficients[i]));
}
return commitments;
}
/**
* Compute verification value (g^{share value}) using coefficient commitments sent by dealer and my share id.
* @param j my share holder id
* @param commitments commitments to polynomial coefficients of share (received from dealer)
* @param group
*
* @return product of Aik ^ (j ^ k) == g ^ polynomial(i)
*/
public static <T> T computeVerificationValue(int j, ArrayList<T> commitments, Group<T> group) {
T v = group.zero();
BigInteger power = BigInteger.ONE;
BigInteger J = BigInteger.valueOf(j);
for (int k = 0 ; k < commitments.size() ; k ++){
v = group.add(v,group.multiply(commitments.get(k),power));
power = power.multiply(J);
}
return v;
}
/**
* getter
* @return generator of group
*/
public T getGenerator() {
return g;
}
/**
* getter
* @return group
*/
public Group<T> getGroup(){
return group;
}
/**
* getter
* @return commitmentsArrayList
*/
public ArrayList<T> getCommitmentsArrayList() {
return commitmentsArrayList;
}
}

66
distributed-key-generation/src/main/java/meerkat/crypto/secretsharing/shamir/LagrangePolynomial.java Normal file
View File

@ -0,0 +1,66 @@
package meerkat.crypto.secretsharing.shamir;
import meerkat.crypto.utils.Arithmetic;
import java.math.BigInteger;
/**
* Created by Tzlil on 1/28/2016.
*
* container of lagrange polynomial
*
* Constructor is private (use {@link #lagrangePolynomials(Polynomial.Point[], Arithmetic)} to construct)
*
* l = (evaluate/divisor)* polynomial
*
* Note : image and divisor stored separately for avoiding lose of information by division
*/
class LagrangePolynomial{
public final Polynomial polynomial;
public final BigInteger image;
public final BigInteger divisor;
/**
* inner constructor, stores all given parameters
* @param polynomial
* @param image
* @param divisor
*/
private LagrangePolynomial(Polynomial polynomial, BigInteger image, BigInteger divisor) {
this.polynomial = polynomial;
this.image = image;
this.divisor = divisor;
}
/**
* static method
* @param points array points s.t there are no couple of points that shares the same x value
*
* @return the lagrange polynomials that mach to given points.
* in case there exists i != j s.t points[i].x == points[j].x returns null.
*/
public static LagrangePolynomial[] lagrangePolynomials(Polynomial.Point[] points,Arithmetic<BigInteger> arithmetic) {
Polynomial one = new Polynomial(new BigInteger[]{BigInteger.ONE},arithmetic);
LagrangePolynomial[] lagrangePolynomials = new LagrangePolynomial[points.length];
Polynomial[] factors = new Polynomial[points.length];
for (int i = 0 ; i < factors.length ; i++){
factors[i] = new Polynomial(new BigInteger[]{points[i].x.negate(),BigInteger.ONE},arithmetic); // X - Xi
}
Polynomial product;
BigInteger divisor;
for(int i = 0; i < points.length; i ++) {
product = one;
divisor = BigInteger.ONE;
for (int j = 0; j < points.length; j++) {
if (i != j) {
divisor = arithmetic.mul(divisor,arithmetic.sub(points[i].x,points[j].x));
product = product.mul(factors[j]);
}
}
if(divisor.equals(BigInteger.ZERO))
return null;
lagrangePolynomials[i] = new LagrangePolynomial(product,points[i].y,divisor);
}
return lagrangePolynomials;
}
}

209
distributed-key-generation/src/main/java/meerkat/crypto/secretsharing/shamir/Polynomial.java Normal file
View File

@ -0,0 +1,209 @@
package meerkat.crypto.secretsharing.shamir;
import meerkat.crypto.utils.Arithmetic;
import java.math.BigInteger;
import java.util.Arrays;
/**
* Created by Tzlil on 1/27/2016.
*/
public class Polynomial implements Comparable<Polynomial> {
private final int degree;
private final BigInteger[] coefficients;
private final Arithmetic<BigInteger> arithmetic;
/**
* constructor
* @param coefficients
* @param arithmetic
* degree set as max index such that coefficients[degree] not equals zero
*/
public Polynomial(BigInteger[] coefficients,Arithmetic<BigInteger> arithmetic) {
int d = coefficients.length - 1;
while (d > 0 && coefficients[d].equals(BigInteger.ZERO)){
d--;
}
this.degree = d;
this.coefficients = coefficients;
this.arithmetic = arithmetic;
}
/**
* Compare to another polynomial (order by degree, then coefficients).
*/
@Override
public int compareTo(Polynomial other) {
if (this.degree != other.degree)
return this.degree - other.degree;
int compare;
for (int i = degree; i >= degree ; i--){
compare = this.coefficients[i].compareTo(other.coefficients[i]);
if (compare != 0){
return compare;
}
}
return 0;
}
/**
* @param x
* @return sum of coefficients[i] * (x ^ i)
*/
public BigInteger evaluate(BigInteger x){
BigInteger result = BigInteger.ZERO;
BigInteger power = BigInteger.ONE;
for(int i = 0 ; i <= degree ; i++){
result = arithmetic.add(result,arithmetic.mul(coefficients[i],power));
power = power.multiply(x);
}
return result;
}
/**
* @param points
* @return polynomial of minimal degree which goes through all points.
* If there exists i != j s.t points[i].x == points[j].x, method returns null.
*/
public static Polynomial interpolation(Point[] points, Arithmetic<BigInteger> arithmetic) {
LagrangePolynomial[] l = LagrangePolynomial.lagrangePolynomials(points,arithmetic);
if (l == null){
return null;
}
// product = product of l[i].divisor
BigInteger product = BigInteger.ONE;
for (int i = 0; i < l.length;i++){
product = arithmetic.mul(product,l[i].divisor);
}
// factor[i] = product divided by l[i].divisor = product of l[j].divisor s.t j!=i
BigInteger[] factors = new BigInteger[l.length];
for (int i = 0; i < l.length;i++){
factors[i] = arithmetic.div(product,l[i].divisor);
}
int degree = l[0].polynomial.degree;
// coefficients[j] = (sum of l[i].evaluate * factor[i] * l[i].coefficients[j] s.t i!=j) divide by product =
// = sum of l[i].evaluate * l[i].coefficients[j] / l[i].divisor s.t i!=j
BigInteger[] coefficients = new BigInteger[degree + 1];
for (int j = 0; j < coefficients.length;j++){
coefficients[j] = BigInteger.ZERO;
for (int i = 0; i < l.length; i++){
BigInteger current = arithmetic.mul(l[i].image,factors[i]);
current = arithmetic.mul(current,l[i].polynomial.coefficients[j]);
coefficients[j] = arithmetic.add(coefficients[j],current);
}
coefficients[j] = arithmetic.div(coefficients[j],product);
}
return new Polynomial(coefficients,arithmetic);
}
/**
* @param other
* @return new Polynomial of degree max(this degree,other degree) s.t for all x
* new.evaluate(x) = this.evaluate(x) + other.evaluate(x)
*/
public Polynomial add(Polynomial other){
Polynomial bigger,smaller;
if(this.degree < other.degree){
bigger = other;
smaller = this;
}else{
bigger = this;
smaller = other;
}
BigInteger[] coefficients = bigger.getCoefficients();
for (int i = 0; i <= smaller.degree ; i++){
coefficients[i] = arithmetic.add(smaller.coefficients[i],bigger.coefficients[i]);
}
return new Polynomial(coefficients,other.arithmetic);
}
/**
* @param constant
* @return new Polynomial of degree this.degree s.t for all x
* new.evaluate(x) = constant * this.evaluate(x)
*/
public Polynomial mul(BigInteger constant){
BigInteger[] coefficients = this.getCoefficients();
for (int i = 0; i <= this.degree ; i++){
coefficients[i] = arithmetic.mul(constant,coefficients[i]);
}
return new Polynomial(coefficients,arithmetic);
}
/**
* @param other
* @return new Polynomial of degree this degree + other degree + 1 s.t for all x
* new.evaluate(x) = this.evaluate(x) * other.evaluate(x)
*/
public Polynomial mul(Polynomial other){
BigInteger[] coefficients = new BigInteger[this.degree + other.degree + 1];
Arrays.fill(coefficients,BigInteger.ZERO);
for (int i = 0; i <= this.degree ; i++){
for (int j = 0; j <= other.degree; j++){
coefficients[i+j] = arithmetic.add(coefficients[i+j],arithmetic.mul(this.coefficients[i],other.coefficients[j]));
}
}
return new Polynomial(coefficients,arithmetic);
}
/** getter
* @return copy of coefficients
*/
public BigInteger[] getCoefficients() {
return Arrays.copyOf(coefficients,coefficients.length);
}
/** getter
* @return degree
*/
public int getDegree() {
return degree;
}
/**
* inner class
* container for (x,y) x from range and y from evaluate of polynomial
*/
public static class Point implements java.io.Serializable {
public final BigInteger x;
public final BigInteger y;
/**
* constructor
* @param x
* @param polynomial y = polynomial.evaluate(x)
*/
public Point(BigInteger x, Polynomial polynomial) {
this.x = x;
this.y = polynomial.evaluate(x);
}
/**
* constructor
* @param x
* @param y
*/
public Point(BigInteger x,BigInteger y) {
this.x = x;
this.y = y;
}
@Override
public boolean equals(Object obj) {
//TODO: is this implementation correct? cannot understand its logic (hai)
if(!super.equals(obj))
return false;
Point other = (Point)obj;
return this.x.equals(other.x) && this.y.equals(other.y);
}
}
}

123
distributed-key-generation/src/main/java/meerkat/crypto/secretsharing/shamir/SecretSharing.java Normal file
View File

@ -0,0 +1,123 @@
package meerkat.crypto.secretsharing.shamir;
import meerkat.crypto.utils.Arithmetic;
import meerkat.crypto.utils.concrete.Fp;
import java.math.BigInteger;
import java.util.Random;
/**
* Created by Tzlil on 1/27/2016.
* an implementation of Shamire's secret sharing scheme
*/
public class SecretSharing{
/**
* threshold
*/
protected final int t;
/**
* number of shares
*/
protected final int n;
/**
* a large prime
*/
protected final BigInteger q;
/**
* random polynomial of degree s.t polynomial.evaluate(0) = secret
*/
protected final Polynomial polynomial;
/**
* constructor
* @param q a large prime.
* @param t threshold. Any t+1 share holders can recover the secret,
* but any set of at most t share holders cannot
* @param n number of share holders
* @param zi secret, chosen from Zq
* @param random use for generate random polynomial
*/
public SecretSharing(int t, int n, BigInteger zi, Random random, BigInteger q) {
this.q = q;
this.t = t;
this.n = n;
this.polynomial = generateRandomPolynomial(zi,random);
}
/**
* @param x
* @param random
* @return new Polynomial polynomial of degree t ,such that
* 1. polynomial(0) = x
* 2. polynomial coefficients randomly chosen from Zq (except of coefficients[0] = x)
*/
private Polynomial generateRandomPolynomial(BigInteger x, Random random) {
BigInteger[] coefficients = new BigInteger[t + 1];
coefficients[0] = x.mod(q);
int bits = q.bitLength();
for (int i = 1 ; i <= t; i++ ){
coefficients[i] = new BigInteger(bits,random).mod(q);
}
return new Polynomial(coefficients,new Fp(q));
}
/**
* @param i in range of [1,...n]
*
* @return polynomial.evaluate(i)
*/
public Polynomial.Point getShare(int i){
assert (i > 0 && i <= n);
return new Polynomial.Point(BigInteger.valueOf(i), polynomial);
}
/**
* @param shares - subset of the original shares
*
* @return evaluate of interpolation(shares) at x = 0
*/
public static BigInteger recoverSecret(Polynomial.Point[] shares, Arithmetic<BigInteger> arithmetic) {
return recoverPolynomial(shares,arithmetic).evaluate(BigInteger.ZERO);
}
/**
* @param shares - subset of the original shares
*
* @return interpolation(shares)
*/
public static Polynomial recoverPolynomial(Polynomial.Point[] shares, Arithmetic<BigInteger> arithmetic) {
return Polynomial.interpolation(shares,arithmetic);
}
/**
* getter
* @return threshold
*/
public int getT() {
return t;
}
/**
* getter
* @return number of share holders
*/
public int getN() {
return n;
}
/**
* getter
* @return the prime was given in the constructor
*/
public BigInteger getQ() {
return q;
}
/**
* getter
* @return the polynomial was generated in constructor
*/
public Polynomial getPolynomial() {
return polynomial;
}
}

38
distributed-key-generation/src/main/java/meerkat/crypto/utils/Arithmetic.java Normal file
View File

@ -0,0 +1,38 @@
package meerkat.crypto.utils;
/**
* Created by Tzlil on 3/17/2016.
* defines the properties of the traditional operations : add,sub,mul,div
* between two objects of type T
*/
public interface Arithmetic<T> {
/**
* addition
* @param a
* @param b
* @return a + b
*/
T add(T a, T b);
/**
* subtraction
* @param a
* @param b
* @return a - b
*/
T sub(T a, T b);
/**
* multiplication
* @param a
* @param b
* @return a * b
*/
T mul(T a, T b);
/**
* division
* @param a
* @param b
* @return a / b
*/
T div(T a, T b);
}

44
distributed-key-generation/src/main/java/meerkat/crypto/utils/concrete/Fp.java Normal file
View File

@ -0,0 +1,44 @@
package meerkat.crypto.utils.concrete;
import meerkat.crypto.utils.Arithmetic;
import org.factcenter.qilin.primitives.concrete.Zpstar;
import java.math.BigInteger;
/**
* Created by Tzlil on 3/17/2016.
* an implementation of Arithmetic<BigInteger> over prime fields: integers modulo p
*/
public class Fp implements Arithmetic<BigInteger> {
public final BigInteger p;
private final Zpstar zp;
/**
* constructor
* @param p prime
*/
public Fp(BigInteger p) {
this.p = p;
this.zp = new Zpstar(p);
}
@Override
public BigInteger add(BigInteger a, BigInteger b){
return a.add(b).mod(p);
}
@Override
public BigInteger sub(BigInteger a, BigInteger b){
return a.add(p).subtract(b).mod(p);
}
@Override
public BigInteger mul(BigInteger a, BigInteger b){
return zp.add(a,b);
}
@Override
public BigInteger div(BigInteger a, BigInteger b){
return mul(a,zp.negate(b));
}
}

46
distributed-key-generation/src/main/proto/meerkat/DKG.proto Normal file
View File

@ -0,0 +1,46 @@
syntax = "proto3";
package meerkat;
option java_package = "meerkat.protobuf";
message Payload {
enum Type {
SHARE = 0;
COMMITMENT = 1;
COMPLAINT = 2;
DONE = 3;
ANSWER = 4;
YCOMMITMENT = 5;
YCOMPLAINT = 6;
YANSWER = 7;
ABORT = 8;
}
// Type of message in protocol
Type type = 1;
oneof payload_data {
IDMessage id = 5;
ShareMessage share = 6;
CommitmentMessage commitment = 7;
}
}
message IDMessage {
int32 id = 1;
}
message ShareMessage {
int32 i = 1;
int32 j = 2;
bytes share = 3;
// For double shares (used in GJKR protocol)
bytes share_t = 4;
}
message CommitmentMessage {
int32 k = 1;
bytes commitment = 2;
}

73
distributed-key-generation/src/test/java/meerkat/crypto/dkg/feldman/DKGMaliciousUser.java Normal file
View File

@ -0,0 +1,73 @@
package meerkat.crypto.dkg.feldman;
import meerkat.comm.Channel;
import java.math.BigInteger;
import java.util.*;
/**
* Created by Tzlil on 3/21/2016.
*/
public class DKGMaliciousUser<T> extends User<T> {
private final Protocol<T> maliciousDkg;
private final Set<Integer> falls;
public DKGMaliciousUser(Protocol<T> dkg, Protocol<T> maliciousDKG, Channel channel, Set<Integer> falls) {
super(dkg, channel);
this.falls = falls;
this.maliciousDkg = maliciousDKG;
maliciousDKG.setParties(parties);
}
public static Set<Integer> selectFallsRandomly(Set<Integer> ids, Random random){
Set<Integer> falls = new HashSet<Integer>();
ArrayList<Integer> idsList = new ArrayList<Integer>();
for (int id : ids){
idsList.add(id);
}
int fallsSize = random.nextInt(idsList.size()) + 1;// 1 - (n-1)
while (falls.size() < fallsSize){
falls.add(idsList.remove(random.nextInt(idsList.size())));
}
return falls;
}
public static <T> Protocol<T> generateMaliciousDKG(Protocol<T> dkg,Channel channel,Random random){
BigInteger q = dkg.getQ();
BigInteger zi = new BigInteger(q.bitLength(), random).mod(q);
Protocol<T> malicious = new Protocol<T>(dkg.getT(),dkg.getN(),zi,random,dkg.getQ()
,dkg.getGenerator(),dkg.getGroup(),dkg.getId(),dkg.getEncoder());
malicious.setChannel(channel);
return malicious;
}
@Override
public void stage1() {
dkg.broadcastCommitments();
sendSecrets(); //insteadof crypto.sendSecrets(channel);
}
@Override
public void stage3() {
maliciousDkg.answerAllComplainingPlayers();
}
@Override
public void stage4(){
// do nothing
}
private void sendSecrets(){
for (int j = 1; j <= n ; j++){
if(j != id){
if(falls.contains(j)){
maliciousDkg.sendSecret(j);
}else {
dkg.sendSecret(j);
}
}
}
}
}

170
distributed-key-generation/src/test/java/meerkat/crypto/dkg/feldman/DKGTest.java Normal file
View File

@ -0,0 +1,170 @@
package meerkat.crypto.dkg.feldman;
import meerkat.comm.ChannelImpl;
import meerkat.crypto.utils.Arithmetic;
import meerkat.crypto.utils.concrete.Fp;
import meerkat.comm.Channel;
import meerkat.crypto.secretsharing.feldman.VerifiableSecretSharing;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import meerkat.crypto.secretsharing.shamir.SecretSharing;
import meerkat.crypto.utils.BigIntegerByteEncoder;
import meerkat.crypto.utils.GenerateRandomPrime;
import org.factcenter.qilin.primitives.Group;
import org.factcenter.qilin.primitives.concrete.Zpstar;
import org.factcenter.qilin.util.ByteEncoder;
import org.junit.Test;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
/**
* Created by Tzlil on 3/21/2016.
*/
public class DKGTest {
int tests = 10;
BigInteger p = GenerateRandomPrime.SafePrime100Bits;
BigInteger q = p.subtract(BigInteger.ONE).divide(BigInteger.valueOf(2));
Group<BigInteger> group = new Zpstar(p);
Arithmetic<BigInteger> arithmetic = new Fp(q);
int t = 9;
int n = 20;
public void oneTest(Testable testable) throws Exception {
for (int i = 0; i < testable.threads.length ; i++){
testable.threads[i].start();
}
for (int i = 0; i < testable.threads.length ; i++){
testable.threads[i].join();
}
// got the right public value
BigInteger publicValue = group.multiply(testable.g,testable.secret);
for (int i: testable.valids){
assert (testable.dkgs[i - 1].getPublicValue().equals(publicValue));
}
// assert valid verification values
BigInteger expected,verification;
for (int i: testable.valids){
expected = group.multiply(testable.g, testable.dkgs[i - 1].getShare().y);
verification = VerifiableSecretSharing.computeVerificationValue(i, testable.dkgs[i - 1].getCommitments(), group);
assert (expected.equals(verification));
}
// restore the secret from shares
ArrayList<Polynomial.Point> sharesList = new ArrayList<Polynomial.Point>();
for (int i: testable.valids){
sharesList.add(testable.dkgs[i - 1].getShare());
}
Polynomial.Point[] shares = new Polynomial.Point[sharesList.size()];
for (int i = 0; i < shares.length; i ++){
shares[i] = sharesList.get(i);
}
BigInteger calculatedSecret = SecretSharing.recoverSecret(shares,arithmetic);
assert (calculatedSecret.equals(testable.secret));
}
@Test
public void test() throws Exception {
Testable testable;
for (int i = 0; i < tests; i++){
testable = new Testable(new Random());
oneTest(testable);
}
}
class Testable{
Set<Integer> valids;
Set<Integer> QUAL;
Set<Integer> aborted;
Set<Integer> malicious;
User<BigInteger>[] dkgs;
Thread[] threads;
BigInteger g;
BigInteger secret;
public Testable(Random random) {
this.dkgs = new User[n];
this.valids = new HashSet<Integer>();
this.QUAL = new HashSet<Integer>();
this.aborted = new HashSet<Integer>();
this.malicious = new HashSet<Integer>();
this.threads = new Thread[n];
this.g = sampleGenerator(random);
ArrayList<Integer> ids = new ArrayList<Integer>();
for (int id = 1; id<= n ; id++){
ids.add(id);
}
int id;
BigInteger s;
Protocol<BigInteger> dkg;
this.secret = BigInteger.ZERO;
ChannelImpl channels = new ChannelImpl();
ByteEncoder<BigInteger> byteEncoder = new BigIntegerByteEncoder();
while (!ids.isEmpty()) {
id = ids.remove(random.nextInt(ids.size()));
Channel channel = channels.getChannel(id);
s = randomIntModQ(random);
dkg = new meerkat.crypto.dkg.feldman.Protocol<BigInteger>(t, n, s, random, q, g, group, id,byteEncoder);
dkgs[id - 1] = randomDKGUser(id,channel,dkg,random);
threads[id - 1] = new Thread(dkgs[id - 1]);
if(QUAL.contains(id)){
this.secret = this.secret.add(s).mod(q);
}
}
}
public User<BigInteger> randomDKGUser(int id, Channel channel, Protocol<BigInteger> dkg, Random random){
if (QUAL.size() <= t) {
valids.add(id);
QUAL.add(id);
return new User<BigInteger>(dkg,channel);
}else{
int type = random.nextInt(3);
switch (type){
case 0:// regular
valids.add(id);
QUAL.add(id);
return new User<BigInteger>(dkg,channel);
case 1:// abort
int abortStage = random.nextInt(2) + 1; // 1 or 2
aborted.add(id);
if (abortStage == 2){
QUAL.add(id);
}
return new DKGUserImplAbort(dkg,channel,abortStage);
case 2:// malicious
malicious.add(id);
Set<Integer> falls = DKGMaliciousUser.selectFallsRandomly(valids,random);
Protocol<BigInteger> maliciousDKG = DKGMaliciousUser.generateMaliciousDKG(dkg,channel,random);
return new DKGMaliciousUser(dkg,maliciousDKG,channel,falls);
default:
return null;
}
}
}
public BigInteger sampleGenerator(Random random){
BigInteger ZERO = group.zero();
BigInteger g;
do {
g = group.sample(random);
} while (!g.equals(ZERO) && !group.multiply(g, q).equals(ZERO));
return g;
}
public BigInteger randomIntModQ(Random random){
return new BigInteger(q.bitLength(), random).mod(q);
}
}
}

65
distributed-key-generation/src/test/java/meerkat/crypto/dkg/feldman/DKGUserImplAbort.java Normal file
View File

@ -0,0 +1,65 @@
package meerkat.crypto.dkg.feldman;
import meerkat.comm.Channel;
import meerkat.protobuf.DKG;
import static meerkat.crypto.dkg.comm.MessageUtils.createMessage;
/**
* Created by Tzlil on 3/14/2016.
*/
public class DKGUserImplAbort<T> extends User<T> {
final int abortStage;
int stage;
public DKGUserImplAbort(Protocol<T> dkg, Channel channel, int abortStage) {
super(dkg, channel);
this.abortStage = abortStage;// 1 - 2
this.stage = 1;
}
private void sendAbort(){
channel.broadcastMessage(createMessage(DKG.Payload.Type.ABORT));
}
@Override
protected void stage1() {
if(stage < abortStage)
super.stage1();
else if(stage == abortStage){
sendAbort();
}
stage++;
}
@Override
protected void stage2() {
if(stage < abortStage)
super.stage2();
else if(stage == abortStage){
sendAbort();
}
stage++;
}
@Override
protected void stage3() {
if(stage < abortStage)
super.stage3();
else if(stage == abortStage){
sendAbort();
}
stage++;
}
@Override
protected void stage4() {
if(stage < abortStage)
super.stage4();
else if(stage == abortStage){
sendAbort();
}
stage++;
}
}

61
distributed-key-generation/src/test/java/meerkat/crypto/dkg/gjkr/SDKGMaliciousUserImpl.java Normal file
View File

@ -0,0 +1,61 @@
package meerkat.crypto.dkg.gjkr;
import meerkat.comm.Channel;
import java.math.BigInteger;
import java.util.Random;
import java.util.Set;
/**
* Created by Tzlil on 3/29/2016.
*/
public class SDKGMaliciousUserImpl<T> extends User<T> {
private final Protocol<T> maliciousSDKG;
private final Set<Integer> falls;
public SDKGMaliciousUserImpl(Protocol<T> sdkg, Protocol<T> maliciousSDKG
, Channel channel, Set<Integer> falls) {
super(sdkg, channel);
this.falls = falls;
this.maliciousSDKG = maliciousSDKG;
maliciousSDKG.setParties(parties);
}
public static<T> Protocol<T> generateMaliciousSDKG(Protocol<T> sdkg,Channel channel,Random random){
BigInteger q = sdkg.getQ();
BigInteger zi = new BigInteger(q.bitLength(), random).mod(q);
Protocol<T> malicious = new Protocol<T>(sdkg.getT(),sdkg.getN(),zi,random,sdkg.getQ()
,sdkg.getGenerator(),sdkg.getH(),sdkg.getGroup(),sdkg.getId(),sdkg.getEncoder());
malicious.setChannel(channel);
return malicious;
}
@Override
public void stage1() {
sdkg.computeAndBroadcastVerificationValues();
sendSecrets(); //insteadof crypto.sendSecrets(channel);
}
@Override
public void stage3() {
maliciousSDKG.answerAllComplainingPlayers();
}
@Override
public void stage4(){
//do nothing
}
private void sendSecrets(){
for (int j = 1; j <= n ; j++){
if(j != id){
if(falls.contains(j)){
maliciousSDKG.sendSecret(j);
}else {
sdkg.sendSecret(j);
}
}
}
}
}

205
distributed-key-generation/src/test/java/meerkat/crypto/dkg/gjkr/SDKGTest.java Normal file
View File

@ -0,0 +1,205 @@
package meerkat.crypto.dkg.gjkr;
import meerkat.comm.ChannelImpl;
import meerkat.crypto.utils.Arithmetic;
import meerkat.crypto.utils.concrete.Fp;
import meerkat.comm.Channel;
import meerkat.crypto.secretsharing.feldman.VerifiableSecretSharing;
import meerkat.crypto.dkg.feldman.DKGMaliciousUser;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import meerkat.crypto.secretsharing.shamir.SecretSharing;
import meerkat.crypto.utils.BigIntegerByteEncoder;
import meerkat.crypto.utils.GenerateRandomPrime;
import meerkat.protobuf.Crypto;
import org.factcenter.qilin.primitives.Group;
import org.factcenter.qilin.primitives.concrete.Zpstar;
import org.factcenter.qilin.util.ByteEncoder;
import org.junit.Assert;
import org.junit.Test;
import org.junit.internal.runners.statements.Fail;
import static org.junit.Assert.*;
import java.math.BigInteger;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* Created by Tzlil on 3/29/2016.
* TODO: Separate into multiple tests,
* TODO: Make tests deterministic (using constant seed for random generator)
*/
public class SDKGTest {
private ExecutorService executorService = Executors.newCachedThreadPool();
final static int NUM_TESTS = 10;
BigInteger p = GenerateRandomPrime.SafePrime100Bits;
BigInteger q = p.subtract(BigInteger.ONE).divide(BigInteger.valueOf(2));
Group<BigInteger> group = new Zpstar(p);
Arithmetic<BigInteger> arithmetic = new Fp(q);
int initialT = 17;
int initialN = 20;
Random rand = new Random(1);
public void oneTest(Testable testable) throws Exception {
for (int i = 0; i < testable.sdkgs.length; i++){
testable.futures[i] = executorService.submit(testable.sdkgs[i]);
}
for (Future ftr : testable.futures) {
ftr.get();
}
// got the right public value
BigInteger publicValue = group.multiply(testable.g, testable.secret);
for (int i : testable.valids){
assertEquals (testable.sdkgs[i-1].getPublicValue(), publicValue);
}
// assert valid verification values
BigInteger expected,verification;
for (int i : testable.valids){
expected = group.multiply(testable.g, testable.sdkgs[i - 1].getShare().y);
verification = VerifiableSecretSharing.computeVerificationValue(i, testable.sdkgs[i - 1].getCommitments(), group);
assertEquals (expected, verification);
}
// restore the secret from shares
ArrayList<Polynomial.Point> sharesList = new ArrayList<>();
for (int i : testable.valids){
sharesList.add(testable.sdkgs[i - 1].getShare());
}
Polynomial.Point[] shares = new Polynomial.Point[sharesList.size()];
shares = sharesList.toArray(shares);
BigInteger calculatedSecret = SecretSharing.recoverSecret(shares, arithmetic);
assertEquals (calculatedSecret, testable.secret);
}
@Test
public void runSharingProtocol() throws Exception {
Testable testable;
for (int i = 0; i < NUM_TESTS; i++) {
testable = new Testable(initialN+i, initialT+i, group, q, rand);
oneTest(testable);
}
}
static class Testable {
Set<Integer> valids;
Set<Integer> QUAL;
Set<Integer> aborted;
Set<Integer> malicious;
User<BigInteger>[] sdkgs;
Future<?>[] futures;
BigInteger g;
BigInteger h;
BigInteger secret;
Group<BigInteger> group;
int n;
int t;
BigInteger q;
Random random;
ChannelImpl channels = new ChannelImpl();
public Testable(int n, int t, Group<BigInteger> group, BigInteger q, Random random) {
this.n = n;
this.t = t;
this.group = group;
this.q = q;
this.random = random;
this.sdkgs = new User[n];
this.valids = new HashSet<>();
this.QUAL = new HashSet<>();
this.aborted = new HashSet<>();
this.malicious = new HashSet<>();
this.futures = new Future[n];
this.g = sampleGenerator(random);
this.h = group.multiply(g, randomIntModQ(random));
List<Integer> ids = new ArrayList<>();
for (int id = 1; id<= n ; id++){
ids.add(id);
}
int id;
BigInteger s;
Channel channel;
Protocol<BigInteger> sdkg;
this.secret = BigInteger.ZERO;
ByteEncoder<BigInteger> encoder = new BigIntegerByteEncoder();
while (!ids.isEmpty()) {
id = ids.remove(random.nextInt(ids.size()));
s = randomIntModQ(random);
channel = channels.getChannel(id);
sdkg = new Protocol<>(t, n, s, random, q, g , h, group, id, encoder);
sdkgs[id - 1] = randomSDKGUser(id, channel, sdkg);
if(QUAL.contains(id)){
this.secret = this.secret.add(s).mod(q);
}
}
}
enum UserType {
HONEST,
FAILSTOP,
MALICIOUS,
}
public User<BigInteger> newSDKGUser(int id, Channel channel, Protocol<BigInteger> sdkg, UserType userType) {
switch(userType) {
case HONEST:
valids.add(id);
QUAL.add(id);
return new User<>(sdkg,channel);
case FAILSTOP:
int abortStage = random.nextInt(3) + 1; // 1 or 2 or 3
aborted.add(id);
if (abortStage > 1){
QUAL.add(id);
}
return new SDKGUserImplAbort(sdkg, channel, abortStage);
case MALICIOUS:
malicious.add(id);
Set<Integer> falls = DKGMaliciousUser.selectFallsRandomly(valids, random);
Protocol<BigInteger> maliciousSDKG = SDKGMaliciousUserImpl.generateMaliciousSDKG(sdkg, channel, random);
return new SDKGMaliciousUserImpl(sdkg, maliciousSDKG, channel, falls);
}
fail("Unknown user type");
return null;
}
public User<BigInteger> randomSDKGUser(int id, Channel channel, Protocol<BigInteger> sdkg){
if (QUAL.size() <= t) {
return newSDKGUser(id, channel, sdkg, UserType.HONEST);
} else {
UserType type = UserType.values()[random.nextInt(UserType.values().length)];
return newSDKGUser(id, channel, sdkg, type);
}
}
public BigInteger sampleGenerator(Random random){
BigInteger ZERO = group.zero();
BigInteger g;
do {
g = group.sample(random);
} while (!g.equals(ZERO) && !group.multiply(g, q).equals(ZERO));
return g;
}
public BigInteger randomIntModQ(Random random){
return new BigInteger(q.bitLength(), random).mod(q);
}
}
}

80
distributed-key-generation/src/test/java/meerkat/crypto/dkg/gjkr/SDKGUserImplAbort.java Normal file
View File

@ -0,0 +1,80 @@
package meerkat.crypto.dkg.gjkr;
import com.google.protobuf.InvalidProtocolBufferException;
import meerkat.crypto.dkg.comm.MessageUtils;
import meerkat.comm.Channel;
import meerkat.protobuf.Comm;
import meerkat.protobuf.DKG;
/**
* Created by Tzlil on 3/14/2016.
*/
public class SDKGUserImplAbort<T> extends User<T> {
public static class AbortException extends RuntimeException {
}
final int abortStage;
int stage;
public SDKGUserImplAbort(Protocol<T> sdkg, Channel channel, int abortStage) {
super(sdkg, channel);
this.abortStage = abortStage;// 1 - 4
this.stage = 1;
}
private void abort(){
//stopReceiver();
channel.broadcastMessage(MessageUtils.createMessage(DKG.Payload.Type.ABORT));
throw new AbortException();
}
@Override
protected void stage1() {
if(stage < abortStage)
super.stage1();
else if(stage == abortStage){
abort();
}
stage++;
}
@Override
protected void stage2() {
if(stage < abortStage)
super.stage2();
else if(stage == abortStage){
abort();
}
stage++;
}
@Override
protected void stage3() {
if(stage < abortStage)
super.stage3();
else if(stage == abortStage){
abort();
}
stage++;
}
@Override
protected void stage4() {
if(stage < abortStage)
super.stage4();
else if(stage == abortStage){
abort();
}
stage++;
}
@Override
public void run() {
try {
super.run();
} catch (AbortException e) {
// Expected
}
}
}

68
distributed-key-generation/src/test/java/meerkat/crypto/secretsharing/feldman/VerifiableSecretSharingTest.java Normal file
View File

@ -0,0 +1,68 @@
package meerkat.crypto.secretsharing.feldman;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import org.factcenter.qilin.primitives.Group;
import org.factcenter.qilin.primitives.concrete.Zpstar;
import org.junit.Before;
import org.junit.Test;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Random;
/**
* Created by Tzlil on 1/29/2016.
*/
public class VerifiableSecretSharingTest {
VerifiableSecretSharing[] verifiableSecretSharingArray;
int tests = 1 << 10;
Random random;
@Before
public void settings(){
BigInteger p = BigInteger.valueOf(2903);
BigInteger q = p.subtract(BigInteger.ONE).divide(BigInteger.valueOf(2));
Zpstar zpstar = new Zpstar(p);
random = new Random();
BigInteger g;
BigInteger ZERO = zpstar.zero();
do{
g = zpstar.sample(random);
}while (!g.equals(ZERO) && !zpstar.multiply(g,q).equals(ZERO));// sample from QRZp*
int t = 8;
int n = 20;
verifiableSecretSharingArray = new VerifiableSecretSharing[tests];
for (int i = 0; i < verifiableSecretSharingArray.length; i++){
verifiableSecretSharingArray[i] = new VerifiableSecretSharing(t,n
,new BigInteger(q.bitLength(),random).mod(q),random,q,g,zpstar);
}
}
public void oneTest(VerifiableSecretSharing<BigInteger> verifiableSecretSharing) throws Exception {
int n = verifiableSecretSharing.getN();
Group<BigInteger> zpstar = verifiableSecretSharing.getGroup();
BigInteger g = verifiableSecretSharing.getGenerator();
Polynomial.Point[] shares = new Polynomial.Point[n];
ArrayList<BigInteger> commitments = verifiableSecretSharing.getCommitmentsArrayList();
BigInteger[] verifications = new BigInteger[n];
for (int i = 1 ; i <= shares.length; i ++){
shares[i - 1] = verifiableSecretSharing.getShare(i);
verifications[i - 1] = VerifiableSecretSharing.computeVerificationValue(i,commitments,zpstar);
}
BigInteger expected;
for (int i = 0 ; i < shares.length ; i++){
expected = zpstar.multiply(g,shares[i].y);
assert (expected.equals(verifications[i]));
}
}
@Test
public void secretSharingTest() throws Exception {
for (int i = 0 ; i < verifiableSecretSharingArray.length; i ++){
oneTest(verifiableSecretSharingArray[i]);
}
}
}

46
distributed-key-generation/src/test/java/meerkat/crypto/secretsharing/shamir/PolynomialTests/AddTest.java Normal file
View File

@ -0,0 +1,46 @@
package meerkat.crypto.secretsharing.shamir.PolynomialTests;
import meerkat.crypto.utils.GenerateRandomPolynomial;
import meerkat.crypto.utils.Z;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import org.junit.Before;
import org.junit.Test;
import java.math.BigInteger;
import java.util.Random;
/**
* Created by Tzlil on 1/27/2016.
*/
public class AddTest {
Polynomial[] arr1;
Polynomial[] arr2;
int tests = 1 << 12;
int maxDegree = 15;
int bits = 128;
Random random;
@Before
public void settings(){
random = new Random();
arr1 = new Polynomial[tests];
arr2 = new Polynomial[tests];
for (int i = 0; i < arr1.length; i++){
arr1[i] = GenerateRandomPolynomial.generateRandomPolynomial(random.nextInt(maxDegree),bits,random,new Z());
arr2[i] = GenerateRandomPolynomial.generateRandomPolynomial(random.nextInt(maxDegree),bits,random,new Z());
}
}
public void oneTest(Polynomial p1, Polynomial p2){
Polynomial sum = p1.add(p2);
BigInteger x = new BigInteger(bits,random);
assert(sum.evaluate(x).equals(p1.evaluate(x).add(p2.evaluate(x))));
}
@Test
public void addTest(){
for (int i = 0 ; i < arr1.length; i ++){
oneTest(arr1[i],arr2[i]);
}
}
}

68
distributed-key-generation/src/test/java/meerkat/crypto/secretsharing/shamir/PolynomialTests/InterpolationTest.java Normal file
View File

@ -0,0 +1,68 @@
package meerkat.crypto.secretsharing.shamir.PolynomialTests;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import meerkat.crypto.utils.Arithmetic;
import meerkat.crypto.utils.concrete.Fp;
import meerkat.crypto.utils.GenerateRandomPolynomial;
import meerkat.crypto.utils.GenerateRandomPrime;
import org.junit.Before;
import org.junit.Test;
import java.math.BigInteger;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
/**
* Created by Tzlil on 1/27/2016.
*/
public class InterpolationTest {
Polynomial[] polynomials;
int tests = 1 << 10;
int maxDegree = 15;
int bits = 128;
Random random;
Polynomial.Point[][] pointsArrays;
Arithmetic<BigInteger> arithmetic;
BigInteger p = GenerateRandomPrime.SafePrime100Bits;
@Before
public void settings(){
random = new Random();
polynomials = new Polynomial[tests];
pointsArrays = new Polynomial.Point[tests][];
arithmetic = new Fp(p);
for (int i = 0; i < polynomials.length; i++){
polynomials[i] = GenerateRandomPolynomial.generateRandomPolynomial(random.nextInt(maxDegree),bits,random,p);
pointsArrays[i] = randomPoints(polynomials[i]);
}
}
public Polynomial.Point[] randomPoints(Polynomial polynomial){
Polynomial.Point[] points = new Polynomial.Point[polynomial.getDegree() + 1];
BigInteger x;
Set<BigInteger> set = new HashSet();
for (int i = 0; i < points.length; i++){
x = new BigInteger(bits,random).mod(p);
if(set.contains(x)){
i--;
continue;
}
set.add(x);
points[i] = new Polynomial.Point(x,polynomial);
}
return points;
}
public void oneTest(Polynomial p, Polynomial.Point[] points) throws Exception {
Polynomial interpolation = Polynomial.interpolation(points,arithmetic);
assert (p.compareTo(interpolation) == 0);
}
@Test
public void interpolationTest() throws Exception {
for (int i = 0; i < polynomials.length; i ++){
oneTest(polynomials[i],pointsArrays[i]);
}
}
}

48
distributed-key-generation/src/test/java/meerkat/crypto/secretsharing/shamir/PolynomialTests/MulByConstTest.java Normal file
View File

@ -0,0 +1,48 @@
package meerkat.crypto.secretsharing.shamir.PolynomialTests;
import meerkat.crypto.utils.GenerateRandomPolynomial;
import meerkat.crypto.utils.Z;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import org.junit.Before;
import org.junit.Test;
import java.math.BigInteger;
import java.util.Random;
/**
* Created by Tzlil on 1/27/2016.
*/
public class MulByConstTest {
Polynomial[] arr1;
BigInteger[] arr2;
int tests = 1 << 12;
int maxDegree = 15;
int bits = 128;
Random random;
@Before
public void settings(){
random = new Random();
arr1 = new Polynomial[tests];
arr2 = new BigInteger[tests];
for (int i = 0; i < arr1.length; i++){
arr1[i] = GenerateRandomPolynomial.generateRandomPolynomial(random.nextInt(maxDegree),bits,random,new Z());
arr2[i] = new BigInteger(bits,random);
}
}
public void oneTest(Polynomial p, BigInteger c){
Polynomial product = p.mul(c);
BigInteger x = new BigInteger(bits,random);
assert(product.evaluate(x).equals(p.evaluate(x).multiply(c)));
}
@Test
public void mulByConstTest(){
for (int i = 0 ; i < arr1.length; i ++){
oneTest(arr1[i],arr2[i]);
}
}
}

48
distributed-key-generation/src/test/java/meerkat/crypto/secretsharing/shamir/PolynomialTests/MulTest.java Normal file
View File

@ -0,0 +1,48 @@
package meerkat.crypto.secretsharing.shamir.PolynomialTests;
import meerkat.crypto.utils.GenerateRandomPolynomial;
import meerkat.crypto.utils.Z;
import meerkat.crypto.secretsharing.shamir.Polynomial;
import org.junit.Before;
import org.junit.Test;
import java.math.BigInteger;
import java.util.Random;
/**
* Created by Tzlil on 1/27/2016.
*/
public class MulTest {
Polynomial[] arr1;
Polynomial[] arr2;
int tests = 1 << 12;
int maxDegree = 15;
int bits = 128;
Random random;
@Before
public void settings(){
random = new Random();
arr1 = new Polynomial[tests];
arr2 = new Polynomial[tests];
for (int i = 0; i < arr1.length; i++){
arr1[i] = GenerateRandomPolynomial.generateRandomPolynomial(random.nextInt(maxDegree),bits,random,new Z());
arr2[i] = GenerateRandomPolynomial.generateRandomPolynomial(random.nextInt(maxDegree),bits,random,new Z());
}
}
public void oneTest(Polynomial p1, Polynomial p2){
Polynomial product = p1.mul(p2);
BigInteger x = new BigInteger(bits,random);
assert(product.evaluate(x).equals(p1.evaluate(x).multiply(p2.evaluate(x))));
}
@Test
public void mulTest(){
for (int i = 0 ; i < arr1.length; i ++){
oneTest(arr1[i],arr2[i]);
}
}
}

64
distributed-key-generation/src/test/java/meerkat/crypto/secretsharing/shamir/SecretSharingTest.java Normal file
View File

@ -0,0 +1,64 @@
package meerkat.crypto.secretsharing.shamir;
import meerkat.crypto.utils.concrete.Fp;
import meerkat.crypto.utils.GenerateRandomPrime;
import org.factcenter.qilin.primitives.CyclicGroup;
import org.factcenter.qilin.primitives.concrete.Zn;
import org.junit.Before;
import org.junit.Test;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
/**
* Created by Tzlil on 1/29/2016.
*/
public class SecretSharingTest {
SecretSharing[] secretSharingArray;
BigInteger[] secrets;
CyclicGroup<BigInteger> group;
int tests = 1 << 10;
Random random;
BigInteger p = GenerateRandomPrime.SafePrime100Bits;
BigInteger q = p.subtract(BigInteger.ONE).divide(BigInteger.valueOf(2));
@Before
public void settings(){
group = new Zn(q);
int t = 9;
int n = 20;
random = new Random();
secretSharingArray = new SecretSharing[tests];
secrets = new BigInteger[tests];
for (int i = 0; i < secretSharingArray.length; i++){
secrets[i] = group.sample(random);
secretSharingArray[i] = new SecretSharing(t,n,secrets[i],random,q);
}
}
public void oneTest(SecretSharing secretSharing, BigInteger secret) throws Exception {
int t = secretSharing.getT();
int n = secretSharing.getN();
Polynomial.Point[] shares = new Polynomial.Point[t + 1];
List<Integer> indexes = new ArrayList<Integer>(n);
for (int i = 1 ; i <= n; i ++){
indexes.add(i);
}
for (int i = 0 ; i < shares.length ; i++){
shares[i] = secretSharing.getShare(indexes.remove(random.nextInt(indexes.size())));
}
BigInteger calculated = SecretSharing.recoverSecret(shares,new Fp(q));
assert (secret.equals(calculated));
}
@Test
public void secretSharingTest() throws Exception {
for (int i = 0 ; i < secretSharingArray.length; i ++){
oneTest(secretSharingArray[i],secrets[i]);
}
}
}

Some files were not shown because too many files have changed in this diff Show More