FS-9775: First tested pass on search functionality, not tested with deep…

FS-9775: First tested pass on search functionality, not tested with deep searching at multiple levels

libs/libks/src/dht/ks_dht-int.h

@@ -258,10 +258,12 @@ KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
 										  int32_t attempts);
 KS_DECLARE(void) ks_dht_job_build_ping(ks_dht_job_t *job, ks_dht_job_callback_t query_callback, ks_dht_job_callback_t finish_callback);
 KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
+										   ks_dht_search_t *search,
 										   ks_dht_job_callback_t query_callback,
 										   ks_dht_job_callback_t finish_callback,
 										   ks_dht_nodeid_t *target);
 KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
+									  ks_dht_search_t *search,
 									  ks_dht_job_callback_t query_callback,
 									  ks_dht_job_callback_t finish_callback,
 									  ks_dht_nodeid_t *target,
@@ -316,14 +318,9 @@ KS_DECLARE(ks_status_t) ks_dht_message_response(ks_dht_message_t *message,
 /**
  *
  */
-KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t *pool, const ks_dht_nodeid_t *target);
+KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t *pool, const ks_dht_nodeid_t *target, ks_dht_search_callback_t callback);
 KS_DECLARE(void) ks_dht_search_destroy(ks_dht_search_t **search);
 
-KS_DECLARE(ks_status_t) ks_dht_search_callback_add(ks_dht_search_t *search, ks_dht_search_callback_t callback);
-KS_DECLARE(void) ks_dht_search_expire(ks_dht_search_t *search, ks_hash_t *pending, int32_t *active);
-
-KS_DECLARE(ks_status_t) ks_dht_search_pending_create(ks_dht_search_pending_t **pending, ks_pool_t *pool, const ks_dht_nodeid_t *nodeid);
-KS_DECLARE(void) ks_dht_search_pending_destroy(ks_dht_search_pending_t **pending);
 
 /**
  *

libs/libks/src/dht/ks_dht.h

@@ -18,7 +18,7 @@ KS_BEGIN_EXTERN_C
 #define KS_DHT_DATAGRAM_BUFFER_SIZE 1000
 
 //#define KS_DHT_RECV_BUFFER_SIZE 0xFFFF
-#define KS_DHT_PULSE_EXPIRATIONS 10
+#define KS_DHT_PULSE_EXPIRATIONS 1
 
 #define KS_DHT_NODEID_SIZE 20
 
@@ -126,6 +126,8 @@ struct ks_dht_job_s {
 
 	enum ks_dht_job_state_t state;
 
+	ks_dht_search_t *search;
+
 	ks_sockaddr_t raddr; // will obtain local endpoint node id when creating message using raddr
 	int32_t attempts;
 
@@ -134,7 +136,6 @@ struct ks_dht_job_s {
 	ks_dht_job_callback_t finish_callback;
 
 	ks_dht_message_t *response;
-	//ks_dht_nodeid_t response_id;
 
 	// job specific query parameters
 	ks_dht_nodeid_t query_target;
@@ -144,6 +145,7 @@ struct ks_dht_job_s {
 	ks_dht_storageitem_t *query_storageitem;
 
 	// job specific response parameters
+	ks_dht_nodeid_t response_id;
 	ks_dht_node_t *response_nodes[KS_DHT_RESPONSE_NODES_MAX_SIZE];
 	ks_size_t response_nodes_count;
 	ks_dht_node_t *response_nodes6[KS_DHT_RESPONSE_NODES_MAX_SIZE];
@@ -214,23 +216,17 @@ struct ks_dht_transaction_s {
 
 struct ks_dht_search_s {
 	ks_pool_t *pool;
-	ks_mutex_t *mutex;
+	ks_dht_search_t *next;
 	ks_dht_nodeid_t target;
-	ks_dht_search_callback_t *callbacks;
-	ks_size_t callbacks_size;
-	ks_hash_t *pending;
-	ks_dht_nodeid_t results[KS_DHT_SEARCH_RESULTS_MAX_SIZE];
+	ks_dht_search_callback_t callback;
+	ks_mutex_t *mutex;
+	ks_hash_t *searched;
+	ks_hash_t *searching;
+	ks_dht_node_t *results[KS_DHT_SEARCH_RESULTS_MAX_SIZE];
 	ks_dht_nodeid_t distances[KS_DHT_SEARCH_RESULTS_MAX_SIZE];
 	ks_size_t results_length;
 };
 
-struct ks_dht_search_pending_s {
-	ks_pool_t *pool;
-	ks_dht_nodeid_t nodeid;
-	ks_time_t expiration;
-	ks_bool_t finished;
-};
-
 struct ks_dht_storageitem_s {
 	ks_pool_t *pool;
 	ks_dht_nodeid_t id;
@@ -283,8 +279,9 @@ struct ks_dht_s {
 	ks_dhtrt_routetable_t *rt_ipv4;
 	ks_dhtrt_routetable_t *rt_ipv6;
 
-	ks_hash_t *searches4_hash;
-	ks_hash_t *searches6_hash;
+	ks_mutex_t *searches_mutex;
+	ks_dht_search_t *searches_first;
+	ks_dht_search_t *searches_last;
 
 	volatile uint32_t token_secret_current;
 	volatile uint32_t token_secret_previous;
@@ -436,12 +433,17 @@ KS_DECLARE(ks_status_t) ks_dht_ping(ks_dht_t *dht, const ks_sockaddr_t *raddr, k
 /**
  *
  */
-KS_DECLARE(ks_status_t) ks_dht_findnode(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_job_callback_t callback, ks_dht_nodeid_t *target);
+KS_DECLARE(ks_status_t) ks_dht_findnode(ks_dht_t *dht,
+										ks_dht_search_t *search,
+										const ks_sockaddr_t *raddr,
+										ks_dht_job_callback_t callback,
+										ks_dht_nodeid_t *target);
 
 /**
  *
  */
 KS_DECLARE(ks_status_t) ks_dht_get(ks_dht_t *dht,
+								   ks_dht_search_t *search,
 								   const ks_sockaddr_t *raddr,
 								   ks_dht_job_callback_t callback,
 								   ks_dht_nodeid_t *target,
@@ -472,11 +474,11 @@ KS_DECLARE(ks_status_t) ks_dht_put(ks_dht_t *dht,
  * @see ks_dht_search_pending_create
  * @see ks_dht_send_findnode
  */
-KS_DECLARE(ks_status_t) ks_dht_search(ks_dht_t *dht,
-									  int32_t family,
-									  ks_dht_nodeid_t *target,
-									  ks_dht_search_callback_t callback,
-									  ks_dht_search_t **search);
+KS_DECLARE(ks_status_t) ks_dht_search_findnode(ks_dht_t *dht,
+											   int32_t family,
+											   ks_dht_nodeid_t *target,
+											   ks_dht_search_callback_t callback,
+											   ks_dht_search_t **search);
 
 
 /**

libs/libks/src/dht/ks_dht_job.c

@@ -45,6 +45,7 @@ KS_DECLARE(void) ks_dht_job_build_ping(ks_dht_job_t *job, ks_dht_job_callback_t
 }
 
 KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
+										   ks_dht_search_t *search,
 										   ks_dht_job_callback_t query_callback,
 										   ks_dht_job_callback_t finish_callback,
 										   ks_dht_nodeid_t *target)
@@ -53,12 +54,14 @@ KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
 	ks_assert(query_callback);
 	ks_assert(target);
 
+	job->search = search;
 	job->query_callback = query_callback;
 	job->finish_callback = finish_callback;
 	job->query_target = *target;
 }
 
 KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
+									  ks_dht_search_t *search,
 									  ks_dht_job_callback_t query_callback,
 									  ks_dht_job_callback_t finish_callback,
 									  ks_dht_nodeid_t *target,
@@ -69,6 +72,7 @@ KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
 	ks_assert(query_callback);
 	ks_assert(target);
 
+	job->search = search;
 	job->query_callback = query_callback;
 	job->finish_callback = finish_callback;
 	job->query_target = *target;

libs/libks/src/dht/ks_dht_search.c

@@ -2,7 +2,7 @@
 #include "ks_dht-int.h"
 #include "sodium.h"
 
-KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t *pool, const ks_dht_nodeid_t *target)
+KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t *pool, const ks_dht_nodeid_t *target, ks_dht_search_callback_t callback)
 {
 	ks_dht_search_t *s;
 	ks_status_t ret = KS_STATUS_SUCCESS;
@@ -21,9 +21,15 @@ KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t
 
 	memcpy(s->target.id, target->id, KS_DHT_NODEID_SIZE);
 
-	ks_hash_create(&s->pending, KS_HASH_MODE_ARBITRARY, KS_HASH_FLAG_RWLOCK, s->pool);
-	ks_assert(s->pending);
-	ks_hash_set_keysize(s->pending, KS_DHT_NODEID_SIZE);
+	s->callback = callback;
+
+	ks_hash_create(&s->searched, KS_HASH_MODE_ARBITRARY, KS_HASH_FLAG_RWLOCK | KS_HASH_FLAG_DUP_CHECK, s->pool);
+	ks_assert(s->searched);
+	ks_hash_set_keysize(s->searched, KS_DHT_NODEID_SIZE);
+
+	ks_hash_create(&s->searching, KS_HASH_MODE_ARBITRARY, KS_HASH_FLAG_RWLOCK | KS_HASH_FLAG_DUP_CHECK, s->pool);
+	ks_assert(s->searching);
+	ks_hash_set_keysize(s->searching, KS_DHT_NODEID_SIZE);
 
 	// done:
 	if (ret != KS_STATUS_SUCCESS) {
@@ -35,85 +41,19 @@ KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t
 KS_DECLARE(void) ks_dht_search_destroy(ks_dht_search_t **search)
 {
 	ks_dht_search_t *s;
-	ks_hash_iterator_t *it;
 
 	ks_assert(search);
 	ks_assert(*search);
 
 	s = *search;
 
-	if (s->pending) {
-		for (it = ks_hash_first(s->pending, KS_UNLOCKED); it; it = ks_hash_next(&it)) {
-			const void *key = NULL;
-			ks_dht_search_pending_t *val = NULL;
-			ks_hash_this(it, &key, NULL, (void **)&val);
-			ks_dht_search_pending_destroy(&val);
-		}
-		ks_hash_destroy(&s->pending);
-	}
-	if (s->callbacks) {
-		ks_pool_free(s->pool, &s->callbacks);
-		s->callbacks = NULL;
-	}
+	if (s->searching) ks_hash_destroy(&s->searching);
+	if (s->searched) ks_hash_destroy(&s->searched);
 	if (s->mutex) ks_mutex_destroy(&s->mutex);
 
 	ks_pool_free(s->pool, search);
 }
 
-KS_DECLARE(ks_status_t) ks_dht_search_callback_add(ks_dht_search_t *search, ks_dht_search_callback_t callback)
-{
-	ks_assert(search);
-
-	if (callback) {
-		int32_t index;
-
-		ks_mutex_lock(search->mutex);
-		index = search->callbacks_size++;
-		search->callbacks = (ks_dht_search_callback_t *)ks_pool_resize(search->pool,
-																	   (void *)search->callbacks,
-																	   sizeof(ks_dht_search_callback_t) * search->callbacks_size);
-		ks_assert(search->callbacks);
-		search->callbacks[index] = callback;
-		ks_mutex_unlock(search->mutex);
-	}
-	return KS_STATUS_SUCCESS;
-}
-
-KS_DECLARE(ks_status_t) ks_dht_search_pending_create(ks_dht_search_pending_t **pending, ks_pool_t *pool, const ks_dht_nodeid_t *nodeid)
-{
-	ks_dht_search_pending_t *p;
-	ks_status_t ret = KS_STATUS_SUCCESS;
-
-	ks_assert(pending);
-	ks_assert(pool);
-
-	*pending = p = ks_pool_alloc(pool, sizeof(ks_dht_search_pending_t));
-	ks_assert(p);
-
-	p->pool = pool;
-	p->nodeid = *nodeid;
-	p->expiration = ks_time_now() + ((ks_time_t)KS_DHT_SEARCH_EXPIRATION * KS_USEC_PER_SEC);
-	p->finished = KS_FALSE;
-
-	// done:
-	if (ret != KS_STATUS_SUCCESS) {
-		if (p) ks_dht_search_pending_destroy(pending);
-	}
-	return ret;
-}
-
-KS_DECLARE(void) ks_dht_search_pending_destroy(ks_dht_search_pending_t **pending)
-{
-	ks_dht_search_pending_t *p;
-
-	ks_assert(pending);
-	ks_assert(*pending);
-
-	p = *pending;
-
-	ks_pool_free(p->pool, pending);
-}
-
 /* For Emacs:
  * Local Variables:
  * mode:c

libs/libks/test/testdht2.c

@@ -32,6 +32,12 @@ ks_status_t dht2_get_token_callback(ks_dht_t *dht, ks_dht_job_t *job)
 	return KS_STATUS_SUCCESS;
 }
 
+ks_status_t dht2_search_findnode_callback(ks_dht_t *dht, ks_dht_search_t *search)
+{
+	diag("dht2_search_findnode_callback %d\n", search->results_length);
+	return KS_STATUS_SUCCESS;
+}
+
 int main() {
 	//ks_size_t buflen;
   ks_status_t err;
@@ -48,7 +54,7 @@ int main() {
   ks_sockaddr_t raddr1;
   //ks_sockaddr_t raddr2;
   //ks_sockaddr_t raddr3;
-  ks_dht_nodeid_t target;
+  //ks_dht_nodeid_t target;
   //ks_dht_storageitem_t *immutable = NULL;
   //ks_dht_storageitem_t *mutable = NULL;
   //const char *v = "Hello World!";
@@ -147,7 +153,6 @@ int main() {
 	ok(err == KS_STATUS_SUCCESS);
   }
 
-  /*
   diag("Ping test\n");
   
   ks_dht_ping(dht2, &raddr1, NULL); // (QUERYING)
@@ -166,14 +171,35 @@ int main() {
 
   diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
   for (int i = 0; i < 10; ++i) {
-	  //diag("DHT 1\n");
 	  ks_dht_pulse(dht1, 100);
-	  //diag("DHT 2\n");
 	  ks_dht_pulse(dht2, 100);
+	  ks_dht_pulse(dht3, 100);
   }
   ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
-  */
+
+  
+  ks_dht_ping(dht3, &raddr1, NULL); // (QUERYING)
+
+  ks_dht_pulse(dht3, 100); // Send queued ping from dht3 to dht1 (RESPONDING)
   
+  ks_dht_pulse(dht1, 100); // Receive and process ping query from dht3, queue and send ping response
+
+  ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep3->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+
+  ks_dht_pulse(dht3, 100); // Receive and process ping response from dht1 (PROCESSING then COMPLETING)
+
+  ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep1->nodeid) != NULL); // The node should be good, and thus be returned as good
+
+  ks_dht_pulse(dht3, 100); // Call finish callback and purge the job (COMPLETING)
+
+  diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
+  for (int i = 0; i < 20; ++i) {
+	  ks_dht_pulse(dht1, 100);
+	  ks_dht_pulse(dht2, 100);
+	  ks_dht_pulse(dht3, 100);
+  }
+  ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
+
   //diag("Get test\n");
   
 
@@ -205,13 +231,14 @@ int main() {
   ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING)
   */
 
+  /*
   diag("Put test\n");
 
   crypto_sign_keypair(pk.key, sk.key);
 
   ks_dht_storageitem_target_mutable(&pk, NULL, 0, &target);
 
-  ks_dht_get(dht2, &raddr1, dht2_get_token_callback, &target, NULL, 0); // create job
+  ks_dht_get(dht2, NULL, &raddr1, dht2_get_token_callback, &target, NULL, 0); // create job
   
   ks_dht_pulse(dht2, 100); // send get query
 
@@ -225,40 +252,51 @@ int main() {
 
   ks_dht_pulse(dht2, 100); // receive put response
 
-  ks_dht_pulse(dht2, 100); // Call finish callback and purse the job (COMPLETING)
+  ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING)
 
   for (int i = 0; i < 10; ++i) {
-	  //diag("DHT 1\n");
 	  ks_dht_pulse(dht1, 100);
-	  //diag("DHT 2\n");
 	  ks_dht_pulse(dht2, 100);
+	  ks_dht_pulse(dht3, 100);
   }
+  */
 
   // Test bootstrap find_node from dht3 to dht1 to find dht2 nodeid
 
-  //diag("Find_Node test\n");
+  /*
+  diag("Find_Node test\n");
 
-  //ks_dht_findnode(dht3, &raddr1, NULL, &ep2->nodeid);
+  ks_dht_findnode(dht3, NULL, &raddr1, NULL, &ep2->nodeid);
 
-  //ks_dht_pulse(dht3, 100); // Send queued findnode from dht3 to dht1
+  ks_dht_pulse(dht3, 100); // Send queued findnode from dht3 to dht1
 
-  //ks_dht_pulse(dht1, 100); // Receive and process findnode query from dht3, queue and send findnode response
+  ks_dht_pulse(dht1, 100); // Receive and process findnode query from dht3, queue and send findnode response
 
-  //ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep3->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+  ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep3->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
 
-  //ks_dht_pulse(dht3, 100); // Receive and process findnode response from dht1
+  ks_dht_pulse(dht3, 100); // Receive and process findnode response from dht1
+  
+  ks_dht_pulse(dht3, 100); // Call finish callback and purge the job (COMPLETING)
 
-  //ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+  ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
   
-  //diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
-  //for (int i = 0; i < 10; ++i) {
-	  //diag("DHT 1\n");
-	  //ks_dht_pulse(dht1, 100);
-	  //diag("DHT 2\n");
-	  //ks_dht_pulse(dht2, 100);
-  //}
-  //ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
+  diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
+  for (int i = 0; i < 10; ++i) {
+	  ks_dht_pulse(dht1, 100);
+	  ks_dht_pulse(dht2, 100);
+	  ks_dht_pulse(dht3, 100);
+  }
+  ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
+  */
 
+  diag("Search test\n");
+  ks_dht_search_findnode(dht3, AF_INET, &ep2->nodeid, dht2_search_findnode_callback, NULL);
+  diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
+  for (int i = 0; i < 30; ++i) {
+	  ks_dht_pulse(dht1, 100);
+	  ks_dht_pulse(dht2, 100);
+	  ks_dht_pulse(dht3, 100);
+  }
 
   /* Cleanup and shutdown */
   diag("Cleanup\n");