FS-9775: Bug fixes and exposed interface changes while implementing tests for…

FS-9775: Bug fixes and exposed interface changes while implementing tests for get/put which are functional and pass initial tests now. Deep searching needs to be revamped now to complete the full announcing process.

libs/libks/src/dht/ks_dht-int.h

@@ -41,7 +41,7 @@ KS_DECLARE(void) ks_dht_pulse_send(ks_dht_t *dht);
  * @param buffer pointer to the buffer able to contain at least (KS_DHT_NODEID_SIZE * 2) + 1 characters
  * @return The pointer to the front of the populated string buffer
  */
-KS_DECLARE(char *) ks_dht_hexid(ks_dht_nodeid_t *id, char *buffer);
+KS_DECLARE(char *) ks_dht_hex(const uint8_t *data, char *buffer, ks_size_t len);
 
 /**
  * Compacts address information as per the DHT specifications.
@@ -270,9 +270,9 @@ KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
 KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
 									  ks_dht_job_callback_t query_callback,
 									  ks_dht_job_callback_t finish_callback,
-									  ks_dht_nodeid_t *target,
-									  uint8_t *salt,
-									  ks_size_t salt_length);
+									  ks_dht_token_t *token,
+									  int64_t cas,
+									  ks_dht_storageitem_t *item);
 KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job);
 
 
@@ -286,7 +286,33 @@ KS_DECLARE(ks_status_t) ks_dht_endpoint_create(ks_dht_endpoint_t **endpoint,
 											   ks_socket_t sock);
 KS_DECLARE(void) ks_dht_endpoint_destroy(ks_dht_endpoint_t **endpoint);
 
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_message_create(ks_dht_message_t **message,
+											  ks_pool_t *pool,
+											  ks_dht_endpoint_t *endpoint,
+											  const ks_sockaddr_t *raddr,
+											  ks_bool_t alloc_data);
+/**
+ *
+ */
+KS_DECLARE(void) ks_dht_message_destroy(ks_dht_message_t **message);
 
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_message_parse(ks_dht_message_t *message, const uint8_t *buffer, ks_size_t buffer_length);
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_message_response(ks_dht_message_t *message,
+												uint8_t *transactionid,
+												ks_size_t transactionid_length,
+												struct bencode **args);
+
+																								
 /**
  *
  */
@@ -299,19 +325,50 @@ KS_DECLARE(void) ks_dht_search_expire(ks_dht_search_t *search, ks_hash_t *pendin
 KS_DECLARE(ks_status_t) ks_dht_search_pending_create(ks_dht_search_pending_t **pending, ks_pool_t *pool, const ks_dht_nodeid_t *nodeid);
 KS_DECLARE(void) ks_dht_search_pending_destroy(ks_dht_search_pending_t **pending);
 
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_storageitem_target_immutable_internal(struct bencode *value, ks_dht_nodeid_t *target);
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_storageitem_target_mutable_internal(ks_dht_storageitem_pkey_t *pk, struct bencode *salt, ks_dht_nodeid_t *target);
 
 /**
  *
  */
-KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t **item, ks_pool_t *pool, ks_dht_nodeid_t *target, struct bencode *v);
+KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable_internal(ks_dht_storageitem_t **item,
+																	 ks_pool_t *pool,
+																	 ks_dht_nodeid_t *target,
+																	 struct bencode *v,
+																	 ks_bool_t clone_v);
+KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t **item,
+															ks_pool_t *pool,
+															ks_dht_nodeid_t *target,
+															const uint8_t *value,
+															ks_size_t value_length);
+KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable_internal(ks_dht_storageitem_t **item,
+																   ks_pool_t *pool,
+																   ks_dht_nodeid_t *target,
+																   struct bencode *v,
+																   ks_bool_t clone_v,
+																   ks_dht_storageitem_pkey_t *pk,
+																   struct bencode *salt,
+																   ks_bool_t clone_salt,
+																   int64_t sequence,
+																   ks_dht_storageitem_signature_t *signature);
 KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable(ks_dht_storageitem_t **item,
 														  ks_pool_t *pool,
 														  ks_dht_nodeid_t *target,
-														  struct bencode *v,
-														  ks_dht_storageitem_key_t *k,
-														  struct bencode *salt,
+														  const uint8_t *value,
+														  ks_size_t value_length,
+														  ks_dht_storageitem_pkey_t *pk,
+														  const uint8_t *salt,
+														  ks_size_t salt_length,
 														  int64_t sequence,
 														  ks_dht_storageitem_signature_t *signature);
+KS_DECLARE(void) ks_dht_storageitem_update_mutable(ks_dht_storageitem_t *item, struct bencode *v, int64_t sequence, ks_dht_storageitem_signature_t *signature);
 KS_DECLARE(void) ks_dht_storageitem_destroy(ks_dht_storageitem_t **item);
 
 /**

libs/libks/src/dht/ks_dht.h

@@ -29,11 +29,12 @@ KS_BEGIN_EXTERN_C
 #define KS_DHT_MESSAGE_QUERY_MAX_SIZE 20
 #define KS_DHT_MESSAGE_ERROR_MAX_SIZE 256
 
-#define KS_DHT_TRANSACTION_EXPIRATION 30
+#define KS_DHT_TRANSACTION_EXPIRATION 10
 #define KS_DHT_SEARCH_EXPIRATION 10
 #define KS_DHT_SEARCH_RESULTS_MAX_SIZE 8 // @todo replace with KS_DHTRT_BUCKET_SIZE
 
-#define KS_DHT_STORAGEITEM_KEY_SIZE crypto_sign_PUBLICKEYBYTES
+#define KS_DHT_STORAGEITEM_PKEY_SIZE crypto_sign_PUBLICKEYBYTES
+#define KS_DHT_STORAGEITEM_SKEY_SIZE crypto_sign_SECRETKEYBYTES
 #define KS_DHT_STORAGEITEM_SALT_MAX_SIZE 64
 #define KS_DHT_STORAGEITEM_SIGNATURE_SIZE crypto_sign_BYTES
 #define KS_DHT_STORAGEITEM_EXPIRATION 7200
@@ -48,7 +49,8 @@ typedef struct ks_dht_datagram_s ks_dht_datagram_t;
 typedef struct ks_dht_job_s ks_dht_job_t;
 typedef struct ks_dht_nodeid_s ks_dht_nodeid_t;
 typedef struct ks_dht_token_s ks_dht_token_t;
-typedef struct ks_dht_storageitem_key_s ks_dht_storageitem_key_t;
+typedef struct ks_dht_storageitem_pkey_s ks_dht_storageitem_pkey_t;
+typedef struct ks_dht_storageitem_skey_s ks_dht_storageitem_skey_t;
 typedef struct ks_dht_storageitem_signature_s ks_dht_storageitem_signature_t;
 typedef struct ks_dht_message_s ks_dht_message_t;
 typedef struct ks_dht_endpoint_s ks_dht_endpoint_t;
@@ -103,7 +105,6 @@ enum ks_dht_job_state_t {
 	KS_DHT_JOB_STATE_QUERYING,
 	KS_DHT_JOB_STATE_RESPONDING,
 	KS_DHT_JOB_STATE_EXPIRING,
-	KS_DHT_JOB_STATE_PROCESSING,
 	KS_DHT_JOB_STATE_COMPLETING,
 };
 
@@ -133,6 +134,9 @@ struct ks_dht_job_s {
 	// job specific query parameters
 	ks_dht_nodeid_t query_target;
 	struct bencode *query_salt;
+	int64_t query_cas;
+	ks_dht_token_t query_token;
+	ks_dht_storageitem_t *query_storageitem;
 
 	// job specific response parameters
 	ks_dht_node_t *response_nodes[KS_DHT_RESPONSE_NODES_MAX_SIZE];
@@ -158,8 +162,12 @@ struct ks_dhtrt_querynodes_s {
     ks_dht_node_t* nodes[ KS_DHTRT_MAXQUERYSIZE ]; /* out: array of peers (ks_dht_node_t* nodes[incount]) */
 };
 
-struct ks_dht_storageitem_key_s {
-	uint8_t key[KS_DHT_STORAGEITEM_KEY_SIZE];
+struct ks_dht_storageitem_pkey_s {
+	uint8_t key[KS_DHT_STORAGEITEM_PKEY_SIZE];
+};
+
+struct ks_dht_storageitem_skey_s {
+	uint8_t key[KS_DHT_STORAGEITEM_SKEY_SIZE];
 };
 
 struct ks_dht_storageitem_signature_s {
@@ -225,8 +233,9 @@ struct ks_dht_storageitem_s {
 	struct bencode *v;
 	
 	ks_bool_t mutable;
-	ks_dht_storageitem_key_t pk;
-	ks_dht_storageitem_key_t sk;
+	ks_mutex_t *mutex;
+	ks_dht_storageitem_pkey_t pk;
+	ks_dht_storageitem_skey_t sk;
 	struct bencode *salt;
 	int64_t seq;
 	ks_dht_storageitem_signature_t sig;
@@ -361,14 +370,88 @@ KS_DECLARE(ks_status_t) ks_dht_bind(ks_dht_t *dht, const ks_dht_nodeid_t *nodeid
  */
 KS_DECLARE(void) ks_dht_pulse(ks_dht_t *dht, int32_t timeout);
 
+
+KS_DECLARE(char *) ks_dht_hex(const uint8_t *data, char *buffer, ks_size_t len);
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_storageitem_target_immutable(const uint8_t *value, ks_size_t value_length, ks_dht_nodeid_t *target);
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_storageitem_target_mutable(ks_dht_storageitem_pkey_t *pk, const uint8_t *salt, ks_size_t salt_length, ks_dht_nodeid_t *target);
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_storageitem_signature_generate(ks_dht_storageitem_signature_t *sig,
+															  ks_dht_storageitem_skey_t *sk,
+															  const uint8_t *salt,
+															  ks_size_t salt_length,
+															  int64_t sequence,
+															  const uint8_t *value,
+															  ks_size_t value_length);
+												
+/**
+ *
+ */
+KS_DECLARE(void) ks_dht_storageitems_read_lock(ks_dht_t *dht);
+
+/**
+ *
+ */
+KS_DECLARE(void) ks_dht_storageitems_read_unlock(ks_dht_t *dht);
+
+/**
+ *
+ */
+KS_DECLARE(void) ks_dht_storageitems_write_lock(ks_dht_t *dht);
+
+/**
+ *
+ */
+KS_DECLARE(void) ks_dht_storageitems_write_unlock(ks_dht_t *dht);
+
+/**
+ *
+ */
+KS_DECLARE(ks_dht_storageitem_t *) ks_dht_storageitems_find(ks_dht_t *dht, ks_dht_nodeid_t *target);
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_storageitems_insert(ks_dht_t *dht, ks_dht_storageitem_t *item);
+						
+/**
+ *
+ */
 KS_DECLARE(ks_status_t) ks_dht_ping(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_job_callback_t callback);
+
+/**
+ *
+ */
 KS_DECLARE(ks_status_t) ks_dht_findnode(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_job_callback_t callback, ks_dht_nodeid_t *target);
+
+/**
+ *
+ */
 KS_DECLARE(ks_status_t) ks_dht_get(ks_dht_t *dht,
 								   const ks_sockaddr_t *raddr,
 								   ks_dht_job_callback_t callback,
 								   ks_dht_nodeid_t *target,
 								   uint8_t *salt,
 								   ks_size_t salt_length);
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_put(ks_dht_t *dht,
+								   const ks_sockaddr_t *raddr,
+								   ks_dht_job_callback_t callback,
+								   ks_dht_token_t *token,
+								   int64_t cas,
+								   ks_dht_storageitem_t *item);
 						
 /**
  * Create a network search of the closest nodes to a target.
@@ -391,34 +474,6 @@ KS_DECLARE(ks_status_t) ks_dht_search(ks_dht_t *dht,
 									  ks_dht_search_t **search);
 
 
-
-/**
- *
- */
-KS_DECLARE(ks_status_t) ks_dht_message_create(ks_dht_message_t **message,
-											  ks_pool_t *pool,
-											  ks_dht_endpoint_t *endpoint,
-											  const ks_sockaddr_t *raddr,
-											  ks_bool_t alloc_data);
-/**
- *
- */
-KS_DECLARE(void) ks_dht_message_destroy(ks_dht_message_t **message);
-
-/**
- *
- */
-KS_DECLARE(ks_status_t) ks_dht_message_parse(ks_dht_message_t *message, const uint8_t *buffer, ks_size_t buffer_length);
-
-/**
- *
- */
-KS_DECLARE(ks_status_t) ks_dht_message_response(ks_dht_message_t *message,
-												uint8_t *transactionid,
-												ks_size_t transactionid_length,
-												struct bencode **args);
-
-
 /**
  * route table methods
  *

libs/libks/src/dht/ks_dht_job.c

@@ -78,18 +78,20 @@ KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
 KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
 									  ks_dht_job_callback_t query_callback,
 									  ks_dht_job_callback_t finish_callback,
-									  ks_dht_nodeid_t *target,
-									  uint8_t *salt,
-									  ks_size_t salt_length)
+									  ks_dht_token_t *token,
+									  int64_t cas,
+									  ks_dht_storageitem_t *item)
 {
 	ks_assert(job);
 	ks_assert(query_callback);
-	ks_assert(target);
+	ks_assert(token);
+	ks_assert(item);
 
 	job->query_callback = query_callback;
 	job->finish_callback = finish_callback;
-	job->query_target = *target;
-	if (salt && salt_length > 0) job->query_salt = ben_blob(salt, salt_length);
+	job->query_token = *token;
+	job->query_cas = cas;
+	job->query_storageitem = item;
 }
 
 KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job)

libs/libks/src/dht/ks_dht_message.c

@@ -105,7 +105,7 @@ KS_DECLARE(ks_status_t) ks_dht_message_parse(ks_dht_message_t *message, const ui
 
 	memcpy(message->type, yv, yv_len);
 	message->type[yv_len] = '\0';
-	ks_log(KS_LOG_DEBUG, "Message type is '%s'\n", message->type);
+	//ks_log(KS_LOG_DEBUG, "Message type is '%s'\n", message->type);
 
 	return KS_STATUS_SUCCESS;
 }

libs/libks/src/dht/ks_dht_search.c

@@ -44,9 +44,9 @@ KS_DECLARE(void) ks_dht_search_destroy(ks_dht_search_t **search)
 
 	if (s->pending) {
 		for (it = ks_hash_first(s->pending, KS_UNLOCKED); it; it = ks_hash_next(&it)) {
-			ks_dht_search_pending_t *val;
-			
-			ks_hash_this_val(it, (void **)&val);
+			const void *key = NULL;
+			ks_dht_search_pending_t *val = NULL;
+			ks_hash_this(it, &key, NULL, (void **)&val);
 			ks_dht_search_pending_destroy(&val);
 		}
 		ks_hash_destroy(&s->pending);

libs/libks/src/dht/ks_dht_storageitem.c

@@ -2,7 +2,11 @@
 #include "ks_dht-int.h"
 #include "sodium.h"
 
-KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t **item, ks_pool_t *pool, ks_dht_nodeid_t *target, struct bencode *v)
+KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable_internal(ks_dht_storageitem_t **item,
+																	 ks_pool_t *pool,
+																	 ks_dht_nodeid_t *target,
+																	 struct bencode *v,
+																	 ks_bool_t clone_v)
 {
 	ks_dht_storageitem_t *si;
 	ks_status_t ret = KS_STATUS_SUCCESS;
@@ -19,16 +23,9 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t
 	si->id = *target;
 	si->mutable = KS_FALSE;
 	si->expiration = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_EXPIRATION * KS_USEC_PER_SEC);
-	si->v = ben_clone(v);
+	si->v = clone_v ? ben_clone(v) : v;
 	ks_assert(si->v);
 
-	//enc = ben_encode(&enc_len, si->v);
-	//ks_assert(enc);
-	//SHA1_Init(&sha);
-	//SHA1_Update(&sha, enc, enc_len);
-	//SHA1_Final(si->id.id, &sha);
-	//free(enc);
-
 	// done:
 	if (ret != KS_STATUS_SUCCESS) {
 		if (si) ks_dht_storageitem_destroy(item);
@@ -36,14 +33,36 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t
 	return ret;
 }
 
-KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable(ks_dht_storageitem_t **item,
-														  ks_pool_t *pool,
-														  ks_dht_nodeid_t *target,
-														  struct bencode *v,
-														  ks_dht_storageitem_key_t *k,
-														  struct bencode *salt,
-														  int64_t sequence,
-														  ks_dht_storageitem_signature_t *signature)
+KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t **item,
+															ks_pool_t *pool,
+															ks_dht_nodeid_t *target,
+															const uint8_t *value,
+															ks_size_t value_length)
+{
+	struct bencode *v = NULL;
+
+	ks_assert(item);
+	ks_assert(pool);
+	ks_assert(value);
+	ks_assert(value_length > 0);
+	ks_assert(SHA_DIGEST_LENGTH == KS_DHT_NODEID_SIZE);
+
+	v = ben_blob(value, value_length);
+	ks_assert(v);
+	
+	return ks_dht_storageitem_create_immutable_internal(item, pool, target, v, KS_FALSE);
+}
+
+KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable_internal(ks_dht_storageitem_t **item,
+																   ks_pool_t *pool,
+																   ks_dht_nodeid_t *target,
+																   struct bencode *v,
+																   ks_bool_t clone_v,
+																   ks_dht_storageitem_pkey_t *pk,
+																   struct bencode *salt,
+																   ks_bool_t clone_salt,
+																   int64_t sequence,
+																   ks_dht_storageitem_signature_t *signature)
 {
 	ks_dht_storageitem_t *si;
 	ks_status_t ret = KS_STATUS_SUCCESS;
@@ -52,7 +71,7 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable(ks_dht_storageitem_t *
 	ks_assert(pool);
 	ks_assert(v);
 	ks_assert(SHA_DIGEST_LENGTH == KS_DHT_NODEID_SIZE);
-	ks_assert(k);
+	ks_assert(pk);
 	ks_assert(signature);
 
 	*item = si = ks_pool_alloc(pool, sizeof(ks_dht_storageitem_t));
@@ -62,12 +81,15 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable(ks_dht_storageitem_t *
 	si->id = *target;
 	si->mutable = KS_TRUE;
 	si->expiration = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_EXPIRATION * KS_USEC_PER_SEC);
-	si->v = ben_clone(v);
+	si->v = clone_v ? ben_clone(v) : v;
 	ks_assert(si->v);
 
-	si->pk = *k;
+	ks_mutex_create(&si->mutex, KS_MUTEX_FLAG_DEFAULT, si->pool);
+	ks_assert(si->mutex);
+	
+	si->pk = *pk;
 	if (salt) {
-		si->salt = ben_clone(salt);
+		si->salt = clone_salt ? ben_clone(salt) : salt;
 		ks_assert(si->salt);
 	}
 	si->seq = sequence;
@@ -80,6 +102,48 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable(ks_dht_storageitem_t *
 	return ret;
 }
 
+KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable(ks_dht_storageitem_t **item,
+														  ks_pool_t *pool,
+														  ks_dht_nodeid_t *target,
+														  const uint8_t *value,
+														  ks_size_t value_length,
+														  ks_dht_storageitem_pkey_t *pk,
+														  const uint8_t *salt,
+														  ks_size_t salt_length,
+														  int64_t sequence,
+														  ks_dht_storageitem_signature_t *signature)
+{
+	struct bencode *v = NULL;
+	struct bencode *s = NULL;
+
+	ks_assert(item);
+	ks_assert(pool);
+	ks_assert(value);
+	ks_assert(value_length > 0);
+	ks_assert(SHA_DIGEST_LENGTH == KS_DHT_NODEID_SIZE);
+	ks_assert(pk);
+	ks_assert(signature);
+
+	v = ben_blob(value, value_length);
+	if (salt && salt_length > 0) s = ben_blob(salt, salt_length);
+	return ks_dht_storageitem_create_mutable_internal(item, pool, target, v, KS_FALSE, pk, s, KS_FALSE, sequence, signature);
+}
+
+KS_DECLARE(void) ks_dht_storageitem_update_mutable(ks_dht_storageitem_t *item, struct bencode *v, int64_t sequence, ks_dht_storageitem_signature_t *signature)
+{
+	ks_assert(item);
+	ks_assert(v);
+	ks_assert(sequence);
+	ks_assert(signature);
+
+	ks_mutex_lock(item->mutex);
+	ben_free(item->v);
+	item->v = ben_clone(v);
+	item->seq = sequence;
+	item->sig = *signature;
+	ks_mutex_unlock(item->mutex);
+}
+
 /**
  *
  */
@@ -96,6 +160,7 @@ KS_DECLARE(void) ks_dht_storageitem_destroy(ks_dht_storageitem_t **item)
 		ben_free(si->v);
 		si->v = NULL;
 	}
+	if (si->mutex) ks_mutex_destroy(&si->mutex);
 	if (si->salt) {
 		ben_free(si->salt);
 		si->salt = NULL;

libs/libks/test/testdht2.c

@@ -3,14 +3,32 @@
 #include <../dht/ks_dht-int.h>
 #include <tap.h>
 
-#define TEST_DHT1_REGISTER_TYPE_BUFFER "d1:ad2:id20:12345678901234567890e1:q4:ping1:t2:421:y1:ze"
-#define TEST_DHT1_PROCESS_QUERY_PING_BUFFER "d1:ad2:id20:12345678901234567890e1:q4:ping1:t2:421:y1:qe"
+ks_dht_storageitem_skey_t sk;
+ks_dht_storageitem_pkey_t pk;
 
-ks_status_t dht_z_callback(ks_dht_t *dht, ks_dht_message_t *message)
+ks_status_t dht2_put_callback(ks_dht_t *dht, ks_dht_job_t *job)
 {
-	diag("dht_z_callback\n");
-	ok(message->transactionid[0] == '4' && message->transactionid[1] == '2');
-	ks_dht_error(dht, message->endpoint, &message->raddr, message->transactionid, message->transactionid_length, 201, "Generic test error");
+	diag("dht2_put_callback\n");
+	return KS_STATUS_SUCCESS;
+}
+
+ks_status_t dht2_get_token_callback(ks_dht_t *dht, ks_dht_job_t *job)
+{
+	char buf[KS_DHT_TOKEN_SIZE * 2 + 1];
+	const char *v = "Hello World!";
+	size_t v_len = strlen(v);
+	ks_dht_storageitem_signature_t sig;
+	ks_dht_storageitem_t *mutable = NULL;
+	
+	diag("dht2_get_token_callback %s\n", ks_dht_hex(job->response_token.token, buf, KS_DHT_TOKEN_SIZE));
+
+	ks_dht_storageitem_signature_generate(&sig, &sk, NULL, 0, 1, (uint8_t *)v, v_len);
+	// @todo check if exists
+	ks_dht_storageitem_create_mutable(&mutable, dht->pool, &job->query_target, (uint8_t *)v, v_len, &pk, NULL, 0, 1, &sig);
+	mutable->sk = sk;
+	ks_dht_storageitems_insert(dht, mutable);
+	
+	ks_dht_put(dht, &job->raddr, dht2_put_callback, &job->response_token, 0, mutable);
 	return KS_STATUS_SUCCESS;
 }
 
@@ -30,6 +48,27 @@ int main() {
   ks_sockaddr_t raddr1;
   //ks_sockaddr_t raddr2;
   //ks_sockaddr_t raddr3;
+  ks_dht_nodeid_t target;
+  //ks_dht_storageitem_t *immutable = NULL;
+  //ks_dht_storageitem_t *mutable = NULL;
+  //const char *v = "Hello World!";
+  //size_t v_len = strlen(v);
+  //ks_dht_storageitem_skey_t sk; //= { { 0xe0, 0x6d, 0x31, 0x83, 0xd1, 0x41, 0x59, 0x22, 0x84, 0x33, 0xed, 0x59, 0x92, 0x21, 0xb8, 0x0b,
+  //0xd0, 0xa5, 0xce, 0x83, 0x52, 0xe4, 0xbd, 0xf0, 0x26, 0x2f, 0x76, 0x78, 0x6e, 0xf1, 0xc7, 0x4d,
+  //0xb7, 0xe7, 0xa9, 0xfe, 0xa2, 0xc0, 0xeb, 0x26, 0x9d, 0x61, 0xe3, 0xb3, 0x8e, 0x45, 0x0a, 0x22,
+  //0xe7, 0x54, 0x94, 0x1a, 0xc7, 0x84, 0x79, 0xd6, 0xc5, 0x4e, 0x1f, 0xaf, 0x60, 0x37, 0x88, 0x1d } };
+  //ks_dht_storageitem_pkey_t pk; //= { { 0x77, 0xff, 0x84, 0x90, 0x5a, 0x91, 0x93, 0x63, 0x67, 0xc0, 0x13, 0x60, 0x80, 0x31, 0x04, 0xf9,
+  //0x24, 0x32, 0xfc, 0xd9, 0x04, 0xa4, 0x35, 0x11, 0x87, 0x6d, 0xf5, 0xcd, 0xf3, 0xe7, 0xe5, 0x48 } };
+  //uint8_t sk1[KS_DHT_STORAGEITEM_SKEY_SIZE];
+  //uint8_t pk1[KS_DHT_STORAGEITEM_PKEY_SIZE];
+  //ks_dht_storageitem_signature_t sig;
+  //char sk_buf[KS_DHT_STORAGEITEM_SKEY_SIZE * 2 + 1];
+  //char pk_buf[KS_DHT_STORAGEITEM_PKEY_SIZE * 2 + 1];
+  //const char *test1vector = "3:seqi1e1:v12:Hello World!";
+  //const char *test1vector = "4:salt6:foobar3:seqi1e1:v12:Hello World!";
+  //size_t test1vector_len = strlen(test1vector);
+  //uint8_t test1vector_sig[KS_DHT_STORAGEITEM_SIGNATURE_SIZE];
+  //char test1vector_buf[KS_DHT_STORAGEITEM_SIGNATURE_SIZE * 2 + 1];
 
   err = ks_init();
   ok(!err);
@@ -62,9 +101,6 @@ int main() {
   err = ks_dht_create(&dht3, NULL, NULL);
   ok(err == KS_STATUS_SUCCESS);
   
-  
-  ks_dht_register_type(dht1, "z", dht_z_callback);
-  
   if (have_v4) {
     err = ks_addr_set(&addr, v4, KS_DHT_DEFAULT_PORT, AF_INET);
 	ok(err == KS_STATUS_SUCCESS);
@@ -111,30 +147,9 @@ int main() {
 	ok(err == KS_STATUS_SUCCESS);
   }
 
-  //diag("Custom type tests\n");
-  
-  //buflen = strlen(TEST_DHT1_REGISTER_TYPE_BUFFER);
-  //memcpy(dht1->recv_buffer, TEST_DHT1_REGISTER_TYPE_BUFFER, buflen);
-  //dht1->recv_buffer_length = buflen;
-
-  //err = ks_dht_process(dht1, ep1, &raddr);
-  //ok(err == KS_STATUS_SUCCESS);
-
-  //ks_dht_pulse(dht1, 100);
-
-  //ks_dht_pulse(&dht2, 100);
-
-  
-  //buflen = strlen(TEST_DHT1_PROCESS_QUERY_PING_BUFFER);
-  //memcpy(dht1->recv_buffer, TEST_DHT1_PROCESS_QUERY_PING_BUFFER, buflen);
-  //dht1->recv_buffer_length = buflen;
-
-  //err = ks_dht_process(dht1, &raddr);
-  //ok(err == KS_STATUS_SUCCESS);
-
+  /*
   diag("Ping test\n");
   
-  //ks_dht_send_ping(dht2, ep2, &raddr1); // Queue bootstrap ping from dht2 to dht1
   ks_dht_ping(dht2, &raddr1, NULL); // (QUERYING)
 
   ks_dht_pulse(dht2, 100); // Send queued ping from dht2 to dht1 (RESPONDING)
@@ -147,7 +162,7 @@ int main() {
 
   ok(ks_dhtrt_find_node(dht2->rt_ipv4, ep1->nodeid) != NULL); // The node should be good, and thus be returned as good
 
-  ks_dht_pulse(dht2, 100); // (COMPLETING)
+  ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING)
 
   diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
   for (int i = 0; i < 10; ++i) {
@@ -157,31 +172,92 @@ int main() {
 	  ks_dht_pulse(dht2, 100);
   }
   ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
+  */
   
-  // Test bootstrap find_node from dht3 to dht1 to find dht2 nodeid
+  //diag("Get test\n");
+  
+
+  /*
+  ks_dht_storageitem_target_immutable((uint8_t *)v, v_len, &target);
+  ks_dht_storageitem_create_immutable(&immutable, dht1->pool, &target, (uint8_t *)v, v_len);
+  ks_dht_storageitems_insert(dht1, immutable);
+  */
+  
+  /*
+  crypto_sign_keypair(pk.key, sk.key);
 
-  diag("Find_Node test\n");
+  ks_dht_storageitem_signature_generate(&sig, &sk, NULL, 0, 1, (uint8_t *)v, v_len);
+  ks_dht_storageitem_target_mutable(&pk, NULL, 0, &target);
+  ks_dht_storageitem_create_mutable(&mutable, dht1->pool, &target, (uint8_t *)v, v_len, &pk, NULL, 0, 1, &sig);
+  mutable->sk = sk;
+  ks_dht_storageitems_insert(dht1, mutable);
 
-  ks_dht_findnode(dht3, &raddr1, NULL, &ep2->nodeid);
+  ks_dht_get(dht2, &raddr1, dht2_get_callback, &target, NULL, 0);
+ 
+  ks_dht_pulse(dht2, 100); // send get query
 
-  ks_dht_pulse(dht3, 100); // Send queued findnode from dht3 to dht1
+  ks_dht_pulse(dht1, 100); // receive get query and send get response
 
-  ks_dht_pulse(dht1, 100); // Receive and process findnode query from dht3, queue and send findnode response
+  ks_dht_pulse(dht2, 100); // receive get response
 
-  ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep3->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+  ok(ks_dht_storageitems_find(dht2, &target) != NULL); // item should be verified and stored
 
-  ks_dht_pulse(dht3, 100); // Receive and process findnode response from dht1
+  ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING)
+  */
 
-  ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+  diag("Put test\n");
+
+  crypto_sign_keypair(pk.key, sk.key);
+
+  ks_dht_storageitem_target_mutable(&pk, NULL, 0, &target);
+
+  ks_dht_get(dht2, &raddr1, dht2_get_token_callback, &target, NULL, 0); // create job
   
-  diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
+  ks_dht_pulse(dht2, 100); // send get query
+
+  ks_dht_pulse(dht1, 100); // receive get query and send get response
+
+  ks_dht_pulse(dht2, 100); // receive get response
+
+  ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING), send put query
+
+  ks_dht_pulse(dht1, 100); // receive put query and send put response
+
+  ks_dht_pulse(dht2, 100); // receive put response
+
+  ks_dht_pulse(dht2, 100); // Call finish callback and purse the job (COMPLETING)
+
   for (int i = 0; i < 10; ++i) {
 	  //diag("DHT 1\n");
 	  ks_dht_pulse(dht1, 100);
 	  //diag("DHT 2\n");
 	  ks_dht_pulse(dht2, 100);
   }
-  ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
+
+  // Test bootstrap find_node from dht3 to dht1 to find dht2 nodeid
+
+  //diag("Find_Node test\n");
+
+  //ks_dht_findnode(dht3, &raddr1, NULL, &ep2->nodeid);
+
+  //ks_dht_pulse(dht3, 100); // Send queued findnode from dht3 to dht1
+
+  //ks_dht_pulse(dht1, 100); // Receive and process findnode query from dht3, queue and send findnode response
+
+  //ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep3->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+
+  //ks_dht_pulse(dht3, 100); // Receive and process findnode response from dht1
+
+  //ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+  
+  //diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
+  //for (int i = 0; i < 10; ++i) {
+	  //diag("DHT 1\n");
+	  //ks_dht_pulse(dht1, 100);
+	  //diag("DHT 2\n");
+	  //ks_dht_pulse(dht2, 100);
+  //}
+  //ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
 
 
   /* Cleanup and shutdown */