FS-9775: Started working on "put", ran into a bug in job states which is fixed…

FS-9775: Started working on "put", ran into a bug in job states which is fixed now, adjusted find_node response to add nodes to the job

libs/libks/Makefile.am

@@ -3,7 +3,7 @@ EXTRA_DIST =
 SUBDIRS = . test
 AUTOMAKE_OPTIONS = subdir-objects
 
-AM_CFLAGS    += -I$(top_srcdir)/src -I$(top_srcdir)/src/include -I$(top_srcdir)/crypt
+AM_CFLAGS    += -I$(top_srcdir)/src -I$(top_srcdir)/src/include -I$(top_srcdir)/crypt -O0
 AM_CPPFLAGS  = $(AM_CFLAGS)
 
 lib_LTLIBRARIES	  = libks.la

libs/libks/src/dht/ks_dht-int.h

@@ -218,6 +218,7 @@ KS_DECLARE(void) ks_dht_pulse_jobs(ks_dht_t *dht);
 KS_DECLARE(ks_status_t) ks_dht_query_ping(ks_dht_t *dht, ks_dht_job_t *job);
 KS_DECLARE(ks_status_t) ks_dht_query_findnode(ks_dht_t *dht, ks_dht_job_t *job);
 KS_DECLARE(ks_status_t) ks_dht_query_get(ks_dht_t *dht, ks_dht_job_t *job);
+KS_DECLARE(ks_status_t) ks_dht_query_put(ks_dht_t *dht, ks_dht_job_t *job);
 
 KS_DECLARE(void *)ks_dht_process(ks_thread_t *thread, void *data);
 
@@ -260,6 +261,18 @@ KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
 										   ks_dht_job_callback_t query_callback,
 										   ks_dht_job_callback_t finish_callback,
 										   ks_dht_nodeid_t *target);
+KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
+									  ks_dht_job_callback_t query_callback,
+									  ks_dht_job_callback_t finish_callback,
+									  ks_dht_nodeid_t *target,
+									  uint8_t *salt,
+									  ks_size_t salt_length);
+KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
+									  ks_dht_job_callback_t query_callback,
+									  ks_dht_job_callback_t finish_callback,
+									  ks_dht_nodeid_t *target,
+									  uint8_t *salt,
+									  ks_size_t salt_length);
 KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job);
 
 
@@ -290,13 +303,13 @@ KS_DECLARE(void) ks_dht_search_pending_destroy(ks_dht_search_pending_t **pending
 /**
  *
  */
-KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t **item, ks_pool_t *pool, struct bencode *v);
+KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t **item, ks_pool_t *pool, ks_dht_nodeid_t *target, struct bencode *v);
 KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable(ks_dht_storageitem_t **item,
 														  ks_pool_t *pool,
+														  ks_dht_nodeid_t *target,
 														  struct bencode *v,
 														  ks_dht_storageitem_key_t *k,
-														  uint8_t *salt,
-														  ks_size_t salt_length,
+														  struct bencode *salt,
 														  int64_t sequence,
 														  ks_dht_storageitem_signature_t *signature);
 KS_DECLARE(void) ks_dht_storageitem_destroy(ks_dht_storageitem_t **item);

libs/libks/src/dht/ks_dht.h

@@ -22,6 +22,8 @@ KS_BEGIN_EXTERN_C
 
 #define KS_DHT_NODEID_SIZE 20
 
+#define KS_DHT_RESPONSE_NODES_MAX_SIZE 8
+
 #define KS_DHT_MESSAGE_TRANSACTIONID_MAX_SIZE 20
 #define KS_DHT_MESSAGE_TYPE_MAX_SIZE 20
 #define KS_DHT_MESSAGE_QUERY_MAX_SIZE 20
@@ -34,6 +36,7 @@ KS_BEGIN_EXTERN_C
 #define KS_DHT_STORAGEITEM_KEY_SIZE crypto_sign_PUBLICKEYBYTES
 #define KS_DHT_STORAGEITEM_SALT_MAX_SIZE 64
 #define KS_DHT_STORAGEITEM_SIGNATURE_SIZE crypto_sign_BYTES
+#define KS_DHT_STORAGEITEM_EXPIRATION 7200
 
 #define KS_DHT_TOKEN_SIZE SHA_DIGEST_LENGTH
 #define KS_DHT_TOKENSECRET_EXPIRATION 300
@@ -92,6 +95,10 @@ struct ks_dht_node_s {
     ks_rwl_t        *reflock;          
 };
 
+struct ks_dht_token_s {
+	uint8_t token[KS_DHT_TOKEN_SIZE];
+};
+
 enum ks_dht_job_state_t {
 	KS_DHT_JOB_STATE_QUERYING,
 	KS_DHT_JOB_STATE_RESPONDING,
@@ -124,7 +131,16 @@ struct ks_dht_job_s {
 	//ks_dht_nodeid_t response_id;
 
 	// job specific query parameters
-	ks_dht_nodeid_t target;
+	ks_dht_nodeid_t query_target;
+	struct bencode *query_salt;
+
+	// job specific response parameters
+	ks_dht_node_t *response_nodes[KS_DHT_RESPONSE_NODES_MAX_SIZE];
+	ks_size_t response_nodes_count;
+	ks_dht_node_t *response_nodes6[KS_DHT_RESPONSE_NODES_MAX_SIZE];
+	ks_size_t response_nodes6_count;
+	ks_dht_token_t response_token;
+	ks_dht_storageitem_t *response_storageitem;
 };
 
 struct ks_dhtrt_routetable_s {
@@ -142,10 +158,6 @@ struct ks_dhtrt_querynodes_s {
     ks_dht_node_t* nodes[ KS_DHTRT_MAXQUERYSIZE ]; /* out: array of peers (ks_dht_node_t* nodes[incount]) */
 };
 
-struct ks_dht_token_s {
-	uint8_t token[KS_DHT_TOKEN_SIZE];
-};
-
 struct ks_dht_storageitem_key_s {
 	uint8_t key[KS_DHT_STORAGEITEM_KEY_SIZE];
 };
@@ -181,7 +193,7 @@ struct ks_dht_transaction_s {
 	ks_pool_t *pool;
 	ks_dht_job_t *job;
 	uint32_t transactionid;
-	ks_dht_nodeid_t target; // @todo look at moving this into job now
+	//ks_dht_nodeid_t target; // @todo look at moving this into job now
 	ks_dht_job_callback_t callback;
 	ks_time_t expiration;
 	ks_bool_t finished;
@@ -209,14 +221,13 @@ struct ks_dht_search_pending_s {
 struct ks_dht_storageitem_s {
 	ks_pool_t *pool;
 	ks_dht_nodeid_t id;
-	// @todo ks_time_t expiration;
+	ks_time_t expiration;
 	struct bencode *v;
 	
 	ks_bool_t mutable;
 	ks_dht_storageitem_key_t pk;
 	ks_dht_storageitem_key_t sk;
-	uint8_t salt[KS_DHT_STORAGEITEM_SALT_MAX_SIZE];
-	ks_size_t salt_length;
+	struct bencode *salt;
 	int64_t seq;
 	ks_dht_storageitem_signature_t sig;
 };
@@ -352,6 +363,12 @@ KS_DECLARE(void) ks_dht_pulse(ks_dht_t *dht, int32_t timeout);
 
 KS_DECLARE(ks_status_t) ks_dht_ping(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_job_callback_t callback);
 KS_DECLARE(ks_status_t) ks_dht_findnode(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_job_callback_t callback, ks_dht_nodeid_t *target);
+KS_DECLARE(ks_status_t) ks_dht_get(ks_dht_t *dht,
+								   const ks_sockaddr_t *raddr,
+								   ks_dht_job_callback_t callback,
+								   ks_dht_nodeid_t *target,
+								   uint8_t *salt,
+								   ks_size_t salt_length);
 						
 /**
  * Create a network search of the closest nodes to a target.

libs/libks/src/dht/ks_dht_bucket.c

@@ -1469,6 +1469,7 @@ void ks_dhtrt_ping(ks_dhtrt_internal_t *internal, ks_dhtrt_bucket_entry_t *entry
 		   ks_dhtrt_printableid(entry->id,buf), entry->outstanding_pings);
 #endif
     ks_dht_node_t* node = entry->gptr;
+	ks_log(KS_LOG_DEBUG, "Node addr %s %d\n", node->addr.host, node->addr.port);
     ks_dht_ping(internal->dht, &node->addr, NULL);
 
 	return;

libs/libks/src/dht/ks_dht_job.c

@@ -38,6 +38,7 @@ KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
 KS_DECLARE(void) ks_dht_job_build_ping(ks_dht_job_t *job, ks_dht_job_callback_t query_callback, ks_dht_job_callback_t finish_callback)
 {
 	ks_assert(job);
+	ks_assert(query_callback);
 
 	job->query_callback = query_callback;
 	job->finish_callback = finish_callback;
@@ -49,11 +50,46 @@ KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
 										   ks_dht_nodeid_t *target)
 {
 	ks_assert(job);
+	ks_assert(query_callback);
 	ks_assert(target);
 
 	job->query_callback = query_callback;
 	job->finish_callback = finish_callback;
-	job->target = *target;
+	job->query_target = *target;
+}
+
+KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
+									  ks_dht_job_callback_t query_callback,
+									  ks_dht_job_callback_t finish_callback,
+									  ks_dht_nodeid_t *target,
+									  uint8_t *salt,
+									  ks_size_t salt_length)
+{
+	ks_assert(job);
+	ks_assert(query_callback);
+	ks_assert(target);
+
+	job->query_callback = query_callback;
+	job->finish_callback = finish_callback;
+	job->query_target = *target;
+	if (salt && salt_length > 0) job->query_salt = ben_blob(salt, salt_length);
+}
+
+KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
+									  ks_dht_job_callback_t query_callback,
+									  ks_dht_job_callback_t finish_callback,
+									  ks_dht_nodeid_t *target,
+									  uint8_t *salt,
+									  ks_size_t salt_length)
+{
+	ks_assert(job);
+	ks_assert(query_callback);
+	ks_assert(target);
+
+	job->query_callback = query_callback;
+	job->finish_callback = finish_callback;
+	job->query_target = *target;
+	if (salt && salt_length > 0) job->query_salt = ben_blob(salt, salt_length);
 }
 
 KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job)
@@ -65,6 +101,10 @@ KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job)
 
 	j = *job;
 
+	if (j->query_salt) ben_free(j->query_salt);
+	for (int32_t i = 0; i < j->response_nodes_count; ++i) ks_dhtrt_release_node(j->response_nodes[i]);
+	for (int32_t i = 0; i < j->response_nodes6_count; ++i) ks_dhtrt_release_node(j->response_nodes6[i]);
+
 	ks_pool_free(j->pool, job);
 }
 

libs/libks/src/dht/ks_dht_message.c

@@ -88,7 +88,7 @@ KS_DECLARE(ks_status_t) ks_dht_message_parse(ks_dht_message_t *message, const ui
 	memcpy(message->transactionid, tv, tv_len);
 	message->transactionid_length = tv_len;
 	// @todo hex output of transactionid
-	//ks_log(KS_LOG_DEBUG, "Message transaction id is %d\n", *transactionid);
+	//ks_log(KS_LOG_DEBUG, "Message transaction id is %d\n", message->transactionid);
 
     y = ben_dict_get_by_str(message->data, "y");
 	if (!y) {

libs/libks/src/dht/ks_dht_search.c

@@ -92,7 +92,7 @@ KS_DECLARE(ks_status_t) ks_dht_search_pending_create(ks_dht_search_pending_t **p
 
 	p->pool = pool;
 	p->nodeid = *nodeid;
-	p->expiration = ks_time_now() + (KS_DHT_SEARCH_EXPIRATION * 1000);
+	p->expiration = ks_time_now() + ((ks_time_t)KS_DHT_SEARCH_EXPIRATION * KS_USEC_PER_SEC);
 	p->finished = KS_FALSE;
 
 	// done:

libs/libks/src/dht/ks_dht_storageitem.c

@@ -2,12 +2,9 @@
 #include "ks_dht-int.h"
 #include "sodium.h"
 
-KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t **item, ks_pool_t *pool, struct bencode *v)
+KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t **item, ks_pool_t *pool, ks_dht_nodeid_t *target, struct bencode *v)
 {
 	ks_dht_storageitem_t *si;
-	SHA_CTX sha;
-	size_t enc_len = 0;
-	uint8_t *enc = NULL;
 	ks_status_t ret = KS_STATUS_SUCCESS;
 
 	ks_assert(item);
@@ -19,16 +16,18 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t
 	ks_assert(si);
 
 	si->pool = pool;
+	si->id = *target;
 	si->mutable = KS_FALSE;
+	si->expiration = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_EXPIRATION * KS_USEC_PER_SEC);
 	si->v = ben_clone(v);
 	ks_assert(si->v);
 
-	enc = ben_encode(&enc_len, si->v);
-	ks_assert(enc);
-	SHA1_Init(&sha);
-	SHA1_Update(&sha, enc, enc_len);
-	SHA1_Final(si->id.id, &sha);
-	free(enc);
+	//enc = ben_encode(&enc_len, si->v);
+	//ks_assert(enc);
+	//SHA1_Init(&sha);
+	//SHA1_Update(&sha, enc, enc_len);
+	//SHA1_Final(si->id.id, &sha);
+	//free(enc);
 
 	// done:
 	if (ret != KS_STATUS_SUCCESS) {
@@ -39,15 +38,14 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable(ks_dht_storageitem_t
 
 KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable(ks_dht_storageitem_t **item,
 														  ks_pool_t *pool,
+														  ks_dht_nodeid_t *target,
 														  struct bencode *v,
 														  ks_dht_storageitem_key_t *k,
-														  uint8_t *salt,
-														  ks_size_t salt_length,
+														  struct bencode *salt,
 														  int64_t sequence,
 														  ks_dht_storageitem_signature_t *signature)
 {
 	ks_dht_storageitem_t *si;
-	SHA_CTX sha;
 	ks_status_t ret = KS_STATUS_SUCCESS;
 
 	ks_assert(item);
@@ -55,30 +53,25 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable(ks_dht_storageitem_t *
 	ks_assert(v);
 	ks_assert(SHA_DIGEST_LENGTH == KS_DHT_NODEID_SIZE);
 	ks_assert(k);
-	ks_assert(!(!salt && salt_length > 0));
-	ks_assert(!(salt_length > KS_DHT_STORAGEITEM_SIGNATURE_SIZE));
 	ks_assert(signature);
 
 	*item = si = ks_pool_alloc(pool, sizeof(ks_dht_storageitem_t));
 	ks_assert(si);
 
 	si->pool = pool;
+	si->id = *target;
 	si->mutable = KS_TRUE;
+	si->expiration = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_EXPIRATION * KS_USEC_PER_SEC);
 	si->v = ben_clone(v);
 	ks_assert(si->v);
 
-	memcpy(si->pk.key, k->key, KS_DHT_STORAGEITEM_KEY_SIZE);
-	if (salt && salt_length > 0) {
-		memcpy(si->salt, salt, salt_length);
-		si->salt_length = salt_length;
+	si->pk = *k;
+	if (salt) {
+		si->salt = ben_clone(salt);
+		ks_assert(si->salt);
 	}
 	si->seq = sequence;
-	memcpy(si->sig.sig, signature->sig, KS_DHT_STORAGEITEM_SIGNATURE_SIZE);
-
-	SHA1_Init(&sha);
-	SHA1_Update(&sha, si->pk.key, KS_DHT_STORAGEITEM_KEY_SIZE);
-	if (si->salt && si->salt_length > 0) SHA1_Update(&sha, si->salt, si->salt_length);
-	SHA1_Final(si->id.id, &sha);
+	si->sig = *signature;
 
 	// done:
 	if (ret != KS_STATUS_SUCCESS) {
@@ -103,6 +96,10 @@ KS_DECLARE(void) ks_dht_storageitem_destroy(ks_dht_storageitem_t **item)
 		ben_free(si->v);
 		si->v = NULL;
 	}
+	if (si->salt) {
+		ben_free(si->salt);
+		si->salt = NULL;
+	}
 
 	ks_pool_free(si->pool, item);
 }

libs/libks/src/dht/ks_dht_transaction.c

@@ -21,7 +21,7 @@ KS_DECLARE(ks_status_t) ks_dht_transaction_create(ks_dht_transaction_t **transac
 	t->job = job;
 	t->transactionid = transactionid;
 	t->callback = callback;
-	t->expiration = ks_time_now() + (KS_DHT_TRANSACTION_EXPIRATION * 1000);
+	t->expiration = ks_time_now() + ((ks_time_t)KS_DHT_TRANSACTION_EXPIRATION * KS_USEC_PER_SEC);
 
 	// done:
 	if (ret != KS_STATUS_SUCCESS) {

libs/libks/test/testdht2.c

@@ -132,7 +132,6 @@ int main() {
   //err = ks_dht_process(dht1, &raddr);
   //ok(err == KS_STATUS_SUCCESS);
 
-  
   diag("Ping test\n");
   
   //ks_dht_send_ping(dht2, ep2, &raddr1); // Queue bootstrap ping from dht2 to dht1
@@ -163,7 +162,6 @@ int main() {
 
   diag("Find_Node test\n");
 
-  //ks_dht_send_findnode(dht3, ep3, &raddr1, &ep2->nodeid); // Queue findnode from dht3 to dht1
   ks_dht_findnode(dht3, &raddr1, NULL, &ep2->nodeid);
 
   ks_dht_pulse(dht3, 100); // Send queued findnode from dht3 to dht1