Cicrumvent possible segfaults in axTLS

More compatibility

3 files changed, 41 insertions, 10 deletions

diff --git a/libs/nixio/Makefile b/libs/nixio/Makefile
index f8d01b7815..aca2751700 100644
--- a/libs/nixio/Makefile
+++ b/libs/nixio/Makefile
@@ -5,7 +5,8 @@ include ../../build/gccconfig.mk
 AXTLS_VERSION = 1.2.1
 AXTLS_DIR     = axTLS
 AXTLS_FILE    = $(AXTLS_DIR)-$(AXTLS_VERSION).tar.gz
-NIXIO_TLS    ?= axtls
+NIXIO_TLS    ?= openssl
+NIXIO_LDFLAGS =
 
 NIXIO_OBJ = src/nixio.o src/socket.o src/sockopt.o src/bind.o src/address.o \
 	    src/poll.o src/io.o src/file.o src/splice.o src/process.o \
@@ -13,15 +14,20 @@ NIXIO_OBJ = src/nixio.o src/socket.o src/sockopt.o src/bind.o src/address.o \
 
 ifeq ($(NIXIO_TLS),axtls)
 	TLS_CFLAGS = -IaxTLS/{ssl,crypto,config} -include src/openssl-compat.h
-	TLS_LDFLAGS =
 	TLS_DEPENDS = src/openssl-compat.o
 	NIXIO_OBJ += src/openssl-compat.o src/libaxtls.a
 endif
 
 ifeq ($(NIXIO_TLS),openssl)
-	TLS_LDFLAGS = -lssl
+	NIXIO_LDFLAGS += -lssl
 endif
 
+
+ifeq ($(OS),SunOS)
+	NIXIO_LDFLAGS += -lsocket -lnsl -lsendfile
+endif
+
+
 %.o: %.c
 	$(COMPILE) $(NIXIO_CFLAGS) $(LUA_CFLAGS) $(FPIC) -c -o $@ $< 
 
@@ -38,7 +44,7 @@ src/openssl-compat.o: src/libaxtls.a src/openssl-compat.c
 	
 
 compile: $(NIXIO_OBJ)
-	$(LINK) $(SHLIB_FLAGS) $(TLS_LDFLAGS) -o src/nixio.so $(NIXIO_OBJ)
+	$(LINK) $(SHLIB_FLAGS) $(NIXIO_LDFLAGS) -o src/nixio.so $(NIXIO_OBJ)
 	mkdir -p dist$(LUA_LIBRARYDIR)
 	cp src/nixio.so dist$(LUA_LIBRARYDIR)/nixio.so
 
@@ -49,7 +55,7 @@ $(AXTLS_DIR)/.prepared:
 	touch $@
 
 src/libaxtls.a: $(AXTLS_DIR)/.prepared
-	$(MAKE) -C $(AXTLS_DIR) CC=$(CC) CFLAGS="$(CFLAGS) $(EXTRA_CFLAGS) $(FPIC) -Wall -pedantic -I../config -I../ssl -I../crypto" LDFLAGS="$(LDFLAGS)" OS="$(OS)" clean all
+	$(MAKE) -C $(AXTLS_DIR) CC=$(CC) CFLAGS="$(CFLAGS) $(EXTRA_CFLAGS) $(FPIC) '-Dalloca(size)=__builtin_alloca(size)' -Wall -pedantic -I../config -I../ssl -I../crypto" LDFLAGS="$(LDFLAGS)" OS="$(OS)" clean all
 	cp -p $(AXTLS_DIR)/_stage/libaxtls.a src
 
 clean: luaclean
diff --git a/libs/nixio/src/nixio-tls.h b/libs/nixio/src/nixio-tls.h
index 13b400ed52..4c58dd1361 100644
--- a/libs/nixio/src/nixio-tls.h
+++ b/libs/nixio/src/nixio-tls.h
@@ -13,6 +13,7 @@
 typedef struct nixio_tls_socket {
 	SSL		*socket;
 #ifdef WITH_AXTLS
+	char	connected;
 	size_t	pbufsiz;
 	char	*pbufpos;
 	char	*pbuffer;
diff --git a/libs/nixio/src/tls-socket.c b/libs/nixio/src/tls-socket.c
index 693a2a5512..0f504ccbbe 100644
--- a/libs/nixio/src/tls-socket.c
+++ b/libs/nixio/src/tls-socket.c
@@ -27,7 +27,7 @@ static int nixio__tls_sock_perror(lua_State *L, SSL *sock, int code) {
 }
 
 static int nixio__tls_sock_pstatus(lua_State *L, SSL *sock, int code) {
-	if (code == 1) {
+	if (code > 0) {
 		lua_pushboolean(L, 1);
 		return 1;
 	} else {
@@ -45,8 +45,26 @@ static SSL* nixio__checktlssock(lua_State *L) {
 	return sock->socket;
 }
 
+#ifndef WITH_AXTLS
+#define nixio_tls__check_connected(L) ;
+
+#define nixio_tls__set_connected(L, val) ;
+#else
+#define nixio_tls__check_connected(L) \
+	nixio_tls_sock *ctsock = luaL_checkudata(L, 1, NIXIO_TLS_SOCK_META);	\
+	if (!ctsock->connected) {	\
+		lua_pushnil(L);			\
+		lua_pushinteger(L, 1);	\
+		return 2;				\
+	}
+
+#define nixio_tls__set_connected(L, val) \
+((nixio_tls_sock*)luaL_checkudata(L, 1, NIXIO_TLS_SOCK_META))->connected = val;
+#endif /* WITH_AXTLS */
+
 static int nixio_tls_sock_recv(lua_State *L) {
 	SSL *sock = nixio__checktlssock(L);
+	nixio_tls__check_connected(L);
 	int req = luaL_checkinteger(L, 2);
 
 	luaL_argcheck(L, req >= 0, 2, "out of range");
@@ -83,7 +101,7 @@ static int nixio_tls_sock_recv(lua_State *L) {
 		return 1;
 	} else {
 		uint8_t *axbuf;
-		size_t axread;
+		int axread;
 
 		/* while handshake pending */
 		while ((axread = ssl_read(sock, &axbuf)) == SSL_OK);
@@ -150,6 +168,7 @@ static int nixio_tls_sock_recv(lua_State *L) {
 
 static int nixio_tls_sock_send(lua_State *L) {
 	SSL *sock = nixio__checktlssock(L);
+	nixio_tls__check_connected(L);
 	size_t len;
 	ssize_t sent;
 	const char *data = luaL_checklstring(L, 2, &len);
@@ -158,22 +177,27 @@ static int nixio_tls_sock_send(lua_State *L) {
 		lua_pushinteger(L, sent);
 		return 1;
 	} else {
-		return nixio__tls_sock_pstatus(L, sock, len);
+		return nixio__tls_sock_pstatus(L, sock, sent);
 	}
 }
 
 static int nixio_tls_sock_accept(lua_State *L) {
 	SSL *sock = nixio__checktlssock(L);
-	return nixio__tls_sock_pstatus(L, sock, SSL_accept(sock));
+	const int stat = SSL_accept(sock);
+	nixio_tls__set_connected(L, stat == 1);
+	return nixio__tls_sock_pstatus(L, sock, stat);
 }
 
 static int nixio_tls_sock_connect(lua_State *L) {
 	SSL *sock = nixio__checktlssock(L);
-	return nixio__tls_sock_pstatus(L, sock, SSL_connect(sock));
+	const int stat = SSL_connect(sock);
+	nixio_tls__set_connected(L, stat == 1);
+	return nixio__tls_sock_pstatus(L, sock, stat);
 }
 
 static int nixio_tls_sock_shutdown(lua_State *L) {
 	SSL *sock = nixio__checktlssock(L);
+	nixio_tls__set_connected(L, 0);
 	return nixio__tls_sock_pstatus(L, sock, SSL_shutdown(sock));
 }