FS-9775: Initial commit of the first portion of the DHT rewrite for peer review

Code compiles, but not yet functional, unit tests to come soon for current code

libs/libks/Makefile.am

@@ -13,6 +13,7 @@ libks_la_SOURCES += src/ks_time.c src/ks_printf.c src/ks_hash.c src/ks_q.c src/k
 libks_la_SOURCES += src/ks_ssl.c src/kws.c src/ks_rng.c
 libks_la_SOURCES += src/utp/utp_api.cpp src/utp/utp_callbacks.cpp src/utp/utp_hash.cpp src/utp/utp_internal.cpp
 libks_la_SOURCES += src/utp/utp_packedsockaddr.cpp src/utp/utp_utils.cpp src/ks_bencode.c
+libks_la_SOURCES += src/dht/ks_dht.c src/dht/ks_dht_endpoint.c src/dht/ks_dht_nodeid.c
 libks_la_SOURCES += crypt/aeskey.c crypt/aestab.c crypt/sha2.c crypt/twofish.c crypt/aes_modes.c crypt/aescrypt.c crypt/twofish_cfb.c 
 #aes.h aescpp.h brg_endian.h aesopt.h aestab.h brg_types.h sha2.h twofish.h
 
@@ -28,6 +29,7 @@ library_include_HEADERS += src/include/ks_dso.h src/include/ks_dht.h src/include
 library_include_HEADERS += src/include/ks_printf.h src/include/ks_hash.h src/include/ks_ssl.h src/include/kws.h
 library_include_HEADERS += src/utp/utp_internal.h src/utp/utp.h src/utp/utp_types.h src/utp/utp_callbacks.h src/utp/utp_templates.h
 library_include_HEADERS += src/utp/utp_hash.h src/utp/utp_packedsockaddr.h src/utp/utp_utils.h src/include/ks_utp.h
+library_include_HEADERS += src/dht/ks_dht.h src/dht/ks_dht-int.h src/dht/ks_dht_endpoint.h src/dht/ks_dht_endpoint-int.h src/dht/ks_dht_nodeid.h
 
 tests: libks.la
 	$(MAKE) -C test tests

libs/libks/src/dht/ks_dht-int.h

+#ifndef KS_DHT_INT_H
+#define KS_DHT_INT_H
+
+#include "ks.h"
+
+KS_BEGIN_EXTERN_C
+
+
+KS_DECLARE(ks_status_t) ks_dht2_idle(ks_dht2_t *dht);
+KS_DECLARE(ks_status_t) ks_dht2_process(ks_dht2_t *dht, ks_sockaddr_t *raddr);
+
+KS_END_EXTERN_C
+
+#endif /* KS_DHT_INT_H */
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */

libs/libks/src/dht/ks_dht.c

+#include "ks_dht.h"
+#include "ks_dht-int.h"
+#include "ks_dht_endpoint-int.h"
+#include "sodium.h"
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_alloc(ks_dht2_t **dht, ks_pool_t *pool)
+{
+	ks_bool_t pool_alloc = !pool;
+	ks_dht2_t *d;
+
+	ks_assert(dht);
+	
+	if (pool_alloc) ks_pool_open(&pool);
+	*dht = d = ks_pool_alloc(pool, sizeof(ks_dht2_t));
+
+	d->pool = pool;
+	d->pool_alloc = pool_alloc;
+
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_free(ks_dht2_t *dht)
+{
+	ks_pool_t *pool = dht->pool;
+	ks_bool_t pool_alloc = dht->pool_alloc;
+
+	ks_pool_free(pool, dht);
+	if (pool_alloc) {
+		ks_pool_close(&pool);
+	}
+
+	return KS_STATUS_SUCCESS;
+}
+												
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_init(ks_dht2_t *dht, const uint8_t *nodeid)
+{
+	ks_assert(dht);
+
+	if (ks_dht2_nodeid_init(&dht->nodeid, nodeid) != KS_STATUS_SUCCESS) {
+		return KS_STATUS_FAIL;
+	}
+	
+    dht->bind_ipv4 = KS_FALSE;
+	dht->bind_ipv6 = KS_FALSE;
+	
+	dht->endpoints = NULL;
+	dht->endpoints_size = 0;
+	ks_hash_create(&dht->endpoints_hash, KS_HASH_MODE_DEFAULT, KS_HASH_FLAG_RWLOCK, dht->pool);
+	dht->endpoints_poll = NULL;
+	
+	dht->recv_buffer_length = 0;
+	
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_deinit(ks_dht2_t *dht)
+{
+	ks_assert(dht);
+
+	dht->recv_buffer_length = 0;
+	// @todo dht->endpoints_poll deinit
+	// @todo dht->endpoints deinit
+	ks_hash_destroy(&dht->endpoints_hash);
+	dht->bind_ipv4 = KS_FALSE;
+	dht->bind_ipv6 = KS_FALSE;
+	ks_dht2_nodeid_deinit(&dht->nodeid);
+	
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_bind(ks_dht2_t *dht, const ks_sockaddr_t *addr)
+{
+	ks_dht2_endpoint_t *ep;
+	ks_socket_t sock;
+	int32_t epindex;
+	
+	ks_assert(dht);
+	ks_assert(addr);
+	ks_assert(addr->family == AF_INET || addr->family == AF_INET6);
+	ks_assert(addr->port);
+	
+	//if (!addr->port) {
+	//	addr->port = KS_DHT_DEFAULT_PORT;
+	//}
+
+	dht->bind_ipv4 |= addr->family == AF_INET;
+	dht->bind_ipv6 |= addr->family == AF_INET6;
+
+	// @todo start of ks_dht2_endpoint_bind
+	if ((sock = socket(addr->family, SOCK_DGRAM, IPPROTO_UDP)) == KS_SOCK_INVALID) {
+		return KS_STATUS_FAIL;
+	}
+
+	// @todo shouldn't ks_addr_bind take a const addr *?
+	if (ks_addr_bind(sock, (ks_sockaddr_t *)addr) != KS_STATUS_SUCCESS) {
+		ks_socket_close(&sock);
+		return KS_STATUS_FAIL;
+	}
+	
+	if (ks_dht2_endpoint_alloc(&ep, dht->pool) != KS_STATUS_SUCCESS) {
+		ks_socket_close(&sock);
+		return KS_STATUS_FAIL;
+	}
+	
+	if (ks_dht2_endpoint_init(ep, addr, sock) != KS_STATUS_SUCCESS) {
+		ks_dht2_endpoint_free(ep);
+		ks_socket_close(&sock);
+		return KS_STATUS_FAIL;
+	}
+
+	ks_socket_option(ep->sock, SO_REUSEADDR, KS_TRUE);
+	ks_socket_option(ep->sock, KS_SO_NONBLOCK, KS_TRUE);
+
+	// @todo end of ks_dht2_endpoint_bind
+	
+	epindex = dht->endpoints_size++;
+	dht->endpoints = (ks_dht2_endpoint_t **)ks_pool_resize(dht->pool,
+														   (void *)dht->endpoints,
+														   sizeof(ks_dht2_endpoint_t *) * dht->endpoints_size);
+	dht->endpoints[epindex] = ep;
+	ks_hash_insert(dht->endpoints_hash, ep->addr.host, ep);
+	
+	dht->endpoints_poll = (struct pollfd *)ks_pool_resize(dht->pool,
+														  (void *)dht->endpoints_poll,
+														  sizeof(struct pollfd) * dht->endpoints_size);
+	dht->endpoints_poll[epindex].fd = ep->sock;
+	dht->endpoints_poll[epindex].events = POLLIN | POLLERR;
+	
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_pulse(ks_dht2_t *dht, int32_t timeout)
+{
+	int32_t result;
+		
+	ks_assert(dht);
+	ks_assert (timeout >= 0);
+
+	// @todo why was old DHT code checking for poll descriptor resizing here?
+
+	if (timeout == 0) {
+		// @todo deal with default timeout, should return quickly but not hog the CPU polling
+	}
+	
+	result = ks_poll(dht->endpoints_poll, dht->endpoints_size, timeout);
+	if (result < 0) {
+		return KS_STATUS_FAIL;
+	}
+
+	if (result == 0) {
+		ks_dht2_idle(dht);
+		return KS_STATUS_TIMEOUT;
+	}
+
+	for (int32_t i = 0; i < dht->endpoints_size; ++i) {
+		if (dht->endpoints_poll[i].revents & POLLIN) {
+			ks_sockaddr_t raddr = KS_SA_INIT;
+			dht->recv_buffer_length = KS_DHT_RECV_BUFFER_SIZE;
+			
+			raddr.family = dht->endpoints[i]->addr.family;
+			if (ks_socket_recvfrom(dht->endpoints_poll[i].fd, dht->recv_buffer, &dht->recv_buffer_length, &raddr) == KS_STATUS_SUCCESS) {
+				ks_dht2_process(dht, &raddr);
+			}
+		}
+	}
+
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_idle(ks_dht2_t *dht)
+{
+	ks_assert(dht);
+
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_process(ks_dht2_t *dht, ks_sockaddr_t *raddr)
+{
+	ks_assert(dht);
+	ks_assert(raddr);
+
+	return KS_STATUS_SUCCESS;
+}
+
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */

libs/libks/src/dht/ks_dht.h

+#ifndef KS_DHT_H
+#define KS_DHT_H
+
+#include "ks.h"
+#include "ks_bencode.h"
+
+#include "ks_dht_endpoint.h"
+#include "ks_dht_nodeid.h"
+
+KS_BEGIN_EXTERN_C
+
+
+#define KS_DHT_DEFAULT_PORT 5309
+#define KS_DHT_RECV_BUFFER_SIZE 0xFFFF
+
+
+typedef struct ks_dht2_s ks_dht2_t;
+struct ks_dht2_s {
+	ks_pool_t *pool;
+	ks_bool_t pool_alloc;
+
+	ks_dht2_nodeid_t nodeid;
+
+	ks_bool_t bind_ipv4;
+	ks_bool_t bind_ipv6;
+
+	ks_dht2_endpoint_t **endpoints;
+	int32_t endpoints_size;
+	ks_hash_t *endpoints_hash;
+	struct pollfd *endpoints_poll;
+
+	uint8_t recv_buffer[KS_DHT_RECV_BUFFER_SIZE];
+	ks_size_t recv_buffer_length;
+};
+
+
+KS_DECLARE(ks_status_t) ks_dht2_alloc(ks_dht2_t **dht, ks_pool_t *pool);
+KS_DECLARE(ks_status_t) ks_dht2_free(ks_dht2_t *dht);
+
+						
+KS_DECLARE(ks_status_t) ks_dht2_init(ks_dht2_t *dht, const uint8_t *nodeid);
+KS_DECLARE(ks_status_t) ks_dht2_deinit(ks_dht2_t *dht);
+
+
+KS_DECLARE(ks_status_t) ks_dht2_bind(ks_dht2_t *dht, const ks_sockaddr_t *addr);
+KS_DECLARE(ks_status_t) ks_dht2_pulse(ks_dht2_t *dht, int32_t timeout);
+
+KS_END_EXTERN_C
+
+#endif /* KS_DHT_H */
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */

libs/libks/src/dht/ks_dht_endpoint-int.h

+#ifndef KS_DHT_ENDPOINT_INT_H
+#define KS_DHT_ENDPOINT_INT_H
+
+#include "ks.h"
+
+KS_BEGIN_EXTERN_C
+
+KS_DECLARE(ks_status_t) ks_dht2_endpoint_alloc(ks_dht2_endpoint_t **endpoint, ks_pool_t *pool);
+KS_DECLARE(ks_status_t) ks_dht2_endpoint_free(ks_dht2_endpoint_t *endpoint);
+
+KS_DECLARE(ks_status_t) ks_dht2_endpoint_init(ks_dht2_endpoint_t *endpoint, const ks_sockaddr_t *addr, ks_socket_t sock);
+KS_DECLARE(ks_status_t) ks_dht2_endpoint_deinit(ks_dht2_endpoint_t *endpoint);
+						
+KS_END_EXTERN_C
+
+#endif /* KS_DHT_ENDPOINT_H */
+
+
+/* For Emacs:
+* Local Variables:
+* mode:c
+* indent-tabs-mode:t
+* tab-width:4
+* c-basic-offset:4
+* End:
+* For VIM:
+* vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+*/

libs/libks/src/dht/ks_dht_endpoint.c

+#include "ks_dht_endpoint.h"
+#include "ks_dht_endpoint-int.h"
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_endpoint_alloc(ks_dht2_endpoint_t **endpoint, ks_pool_t *pool)
+{
+	ks_dht2_endpoint_t *ep;
+
+	ks_assert(endpoint);
+	ks_assert(pool);
+	
+	*endpoint = ep = ks_pool_alloc(pool, sizeof(ks_dht2_endpoint_t));
+	ep->pool = pool;
+
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_endpoint_free(ks_dht2_endpoint_t *endpoint)
+{
+	ks_assert(endpoint);
+	
+	ks_pool_free(endpoint->pool, endpoint);
+
+	return KS_STATUS_SUCCESS;
+}
+												
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_endpoint_init(ks_dht2_endpoint_t *endpoint, const ks_sockaddr_t *addr, ks_socket_t sock)
+{
+	ks_assert(endpoint);
+	ks_assert(addr);
+	ks_assert(addr->family == AF_INET || addr->family == AF_INET6);
+
+	endpoint->addr = *addr;
+	endpoint->sock = sock;
+	
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_endpoint_deinit(ks_dht2_endpoint_t *endpoint)
+{
+	ks_assert(endpoint);
+
+	return KS_STATUS_SUCCESS;
+}
+
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */

libs/libks/src/dht/ks_dht_endpoint.h

+#ifndef KS_DHT_ENDPOINT_H
+#define KS_DHT_ENDPOINT_H
+
+#include "ks.h"
+
+KS_BEGIN_EXTERN_C
+
+typedef struct ks_dht2_endpoint_s ks_dht2_endpoint_t;
+struct ks_dht2_endpoint_s {
+	ks_pool_t *pool;
+	ks_sockaddr_t addr;
+	ks_socket_t sock;
+};
+
+KS_END_EXTERN_C
+
+#endif /* KS_DHT_ENDPOINT_H */
+
+
+/* For Emacs:
+* Local Variables:
+* mode:c
+* indent-tabs-mode:t
+* tab-width:4
+* c-basic-offset:4
+* End:
+* For VIM:
+* vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+*/

libs/libks/src/dht/ks_dht_nodeid.c

+#include "ks_dht_nodeid.h"
+#include "sodium.h"
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_nodeid_alloc(ks_dht2_nodeid_t **nodeid, ks_pool_t *pool)
+{
+	ks_dht2_nodeid_t *nid;
+
+	ks_assert(nodeid);
+	ks_assert(pool);
+	
+	*nodeid = nid = ks_pool_alloc(pool, sizeof(ks_dht2_nodeid_t));
+	nid->pool = pool;
+
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_nodeid_free(ks_dht2_nodeid_t *nodeid)
+{
+	ks_assert(nodeid);
+	
+	ks_pool_free(nodeid->pool, nodeid);
+
+	return KS_STATUS_SUCCESS;
+}
+												
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_nodeid_init(ks_dht2_nodeid_t *nodeid, const uint8_t *id)
+{
+	ks_assert(nodeid);
+	ks_assert(id);
+
+	if (!id) {
+		randombytes_buf(nodeid->id, KS_DHT_NODEID_LENGTH);
+	} else {
+		memcpy(nodeid->id, id, KS_DHT_NODEID_LENGTH);
+	}
+	
+	return KS_STATUS_SUCCESS;
+}
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht2_nodeid_deinit(ks_dht2_nodeid_t *nodeid)
+{
+	ks_assert(nodeid);
+
+	return KS_STATUS_SUCCESS;
+}
+
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */

libs/libks/src/dht/ks_dht_nodeid.h

+#ifndef KS_DHT_NODEID_H
+#define KS_DHT_NODEID_H
+
+#include "ks.h"
+
+KS_BEGIN_EXTERN_C
+
+#define KS_DHT_NODEID_LENGTH 20
+
+typedef struct ks_dht2_nodeid_s ks_dht2_nodeid_t;
+struct ks_dht2_nodeid_s {
+	ks_pool_t *pool;
+    uint8_t id[KS_DHT_NODEID_LENGTH];
+};
+
+KS_DECLARE(ks_status_t) ks_dht2_nodeid_alloc(ks_dht2_nodeid_t **nodeid, ks_pool_t *pool);
+KS_DECLARE(ks_status_t) ks_dht2_nodeid_free(ks_dht2_nodeid_t *nodeid);
+
+KS_DECLARE(ks_status_t) ks_dht2_nodeid_init(ks_dht2_nodeid_t *nodeid, const uint8_t *id);
+KS_DECLARE(ks_status_t) ks_dht2_nodeid_deinit(ks_dht2_nodeid_t *nodeid);
+
+KS_END_EXTERN_C
+
+#endif /* KS_DHT_NODEID_H */
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */
+