CreateNLSocket and CloseNLSocket should return gpointer (mono#15408)

In managed, these functions are used as `static external IntPtr`. This means that previously on arm64 the top bits of the return value were garbage. A comparison with 64-bit -1 like in LinuxNetworkChange.EnsureSocket would never be true, which was causing us to hit other assertions in the runtime.
Co-authored-by: Aleksey Kliger <alklig@microsoft.com>

Author: CoffeeFlux

mono/metadata/icall-decl.h

@@ -211,9 +211,9 @@ ICALL_EXPORT gint32 ves_icall_System_IO_Compression_DeflateStreamNative_ReadZStr
 ICALL_EXPORT gint32 ves_icall_System_IO_Compression_DeflateStreamNative_WriteZStream (gpointer stream, gpointer buffer, gint32 length);
 #endif
 #if defined(ENABLE_MONODROID)
-ICALL_EXPORT gint32 ves_icall_System_Net_NetworkInformation_LinuxNetworkChange_CreateNLSocket (void);
+ICALL_EXPORT gpointer ves_icall_System_Net_NetworkInformation_LinuxNetworkChange_CreateNLSocket (void);
 ICALL_EXPORT gint32 ves_icall_System_Net_NetworkInformation_LinuxNetworkChange_ReadEvents (gpointer sock, gpointer buffer, gint32 count, gint32 size);
-ICALL_EXPORT gint32 ves_icall_System_Net_NetworkInformation_LinuxNetworkChange_CloseNLSocket (gpointer sock);
+ICALL_EXPORT gpointer ves_icall_System_Net_NetworkInformation_LinuxNetworkChange_CloseNLSocket (gpointer sock);
 #endif
 
 #endif // __MONO_METADATA_ICALL_H__

mono/metadata/icall.c

@@ -9228,11 +9228,11 @@ ves_icall_System_Environment_get_ProcessorCount (void)
 
 #if defined(ENABLE_MONODROID)
 
-G_EXTERN_C gint32 CreateNLSocket (void);
+G_EXTERN_C gpointer CreateNLSocket (void);
 G_EXTERN_C gint32 ReadEvents (gpointer sock, gpointer buffer, gint32 count, gint32 size);
-G_EXTERN_C gint32 CloseNLSocket (gpointer sock);
+G_EXTERN_C gpointer CloseNLSocket (gpointer sock);
 
-gint32
+gpointer
 ves_icall_System_Net_NetworkInformation_LinuxNetworkChange_CreateNLSocket (void)
 {
 	return CreateNLSocket ();
@@ -9244,7 +9244,7 @@ ves_icall_System_Net_NetworkInformation_LinuxNetworkChange_ReadEvents (gpointer
 	return ReadEvents (sock, buffer, count, size);
 }
 
-gint32
+gpointer
 ves_icall_System_Net_NetworkInformation_LinuxNetworkChange_CloseNLSocket (gpointer sock)
 {
 	return CloseNLSocket (sock);

support/nl.c

@@ -163,7 +163,7 @@ value_to_name (value2name_t *tbl, int value)
 }
 #endif /* NL_DEBUG */
 
-int
+gpointer
 CreateNLSocket (void)
 {
 	int sock;
@@ -177,22 +177,22 @@ CreateNLSocket (void)
 		ret |= O_NONBLOCK;
 		ret = fcntl (sock, F_SETFL, ret);
 		if (ret < 0)
-			return -1;
+			return GINT_TO_POINTER (-1);
 	}
 
 	memset (&sa, 0, sizeof (sa));
 	if (sock < 0)
-		return -1;
+		return GINT_TO_POINTER (-1);
 	sa.nl_family = AF_NETLINK;
 	sa.nl_pid = getpid ();
 	sa.nl_groups = RTMGRP_IPV4_ROUTE | RTMGRP_IPV6_ROUTE | RTMGRP_NOTIFY;
 	/* RTNLGRP_IPV4_IFADDR | RTNLGRP_IPV6_IFADDR
 	 * RTMGRP_LINK */
 
 	if (bind (sock, (struct sockaddr *) &sa, sizeof (sa)) < 0)
-		return -1;
+		return GINT_TO_POINTER (-1);
 
-	return sock;
+	return GINT_TO_POINTER (sock);
 }
 
 int
@@ -359,10 +359,10 @@ ReadEvents (gpointer sock, gpointer buffer, gint32 count, gint32 size)
 	return result;
 }
 
-int
+gpointer
 CloseNLSocket (gpointer sock)
 {
-	return close (GPOINTER_TO_INT (sock));
+	return GINT_TO_POINTER (close (GPOINTER_TO_INT (sock)));
 }
 #else
 int
@@ -371,16 +371,16 @@ ReadEvents (gpointer sock, gpointer buffer, gint32 count, gint32 size)
 	return 0;
 }
 
-int
+gpointer
 CreateNLSocket (void)
 {
-	return -1;
+	return GINT_TO_POINTER (-1);
 }
 
-int
+gpointer
 CloseNLSocket (gpointer sock)
 {
-	return -1;
+	return GINT_TO_POINTER (-1);
 }
 #endif /* linux/netlink.h + linux/rtnetlink.h */
 

support/nl.h

@@ -3,9 +3,9 @@
 #include <glib.h>
 
 G_BEGIN_DECLS
-int CreateNLSocket (void);
+gpointer CreateNLSocket (void);
 int ReadEvents (gpointer sock, gpointer buffer, gint32 count, gint32 size);
-int CloseNLSocket (gpointer sock);
+gpointer CloseNLSocket (gpointer sock);
 G_END_DECLS
 
 #endif