【译】使用 SO_REUSEPORT 套接字选项提升服务性能

server.sin_family = AF_INET;
server.sin_port = htons(SERVER_PORT);
bcopy(server_ent->h_addr, &server.sin_addr.s_addr, server_ent->h_length);

/* Connect to server, and set the socket's destination IP address and port#
* based on above parameters. Also, request the kernel to automatically set
* the Source IP and port# if the application did not call bind() prior to connect().
*/
connect(fd, (struct sockaddr *)&server, sizeof server);

srv_addr.sin_family = AF_INET;
srv_addr.sin_addr.s_addr = INADDR_ANY;
srv_addr.sin_port = htons(SERVER_PORT);
bind(fd, &srv_addr, sizeof srv_addr);

const struct linger opt = { .l_onoff = 1, .l_linger = 0 };

setsockopt(fd, SOL_SOCKET, SO_LINGER, &opt, sizeof opt);

close(fd);

1. Create a socket:
server_fd = socket(...);

2. Bind to a well known IP address and port#:
ret = bind(server_fd, ...);

3. Mark the socket as passive by changing it's state to LISTEN:
ret = listen(server_fd, ...);

4. Wait for a client to connect, and get a reference file descriptor:
client_fd = accept(server_fd, ...);

# Server is listening on port #45000
$ ss -tan | grep :45000
LISTEN 0 1 10.20.1.1:45000 *:*
# A client connects to the server using it's source port 54762. A new
# socket is created and is seen in ESTABLISHED state, along with the
# earlier LISTEN socket.

$ ss -tan | grep :45000
LISTEN 0 1 10.20.1.1:45000 *:*
ESTAB 0 0 10.20.1.1:45000 10.20.1.100:54762

# Kill the server application.
$ pkill -9 my_server

# Restart the server application.
$ ./my_server 45000
bind: Address already in use

# Find out why
$ ss -tan | grep :45000
TIME-WAIT 0 0 10.20.1.1:45000 10.20.1.100:54762

_sk = __inet_lookup_skb(&tcp_hashinfo, skb, src_port, dst_port);_

#define LHTABLE_SIZE 32 /* Yes, really, this is all you need */

struct inet_hashinfo {
	/* Hash table for fully established sockets 已经建立好的套接字 */
	struct inet_ehash_bucket *ehash;
	
	/* Hash table for LISTEN sockets 处在 LISTEN 状态的套接字 */
	struct inet_listen_hashbucket listening_hash[LHTABLE_SIZE];

};

struct inet_hashinfo tcp_hashinfo;

struct sock *__inet_lookup_skb(tcp_hashinfo, skb, src_port, dst_port)
{
	/* Get the IPv4 header to know the source and destination IP's */
	const struct iphdr *iph = ip_hdr(skb);
	
	/*
	* Look up the incoming skb in tcp_hashinfo using the
	* [ Source-IP:Port, Destination-IP:Port ] tuple.
	*/
	return __inet_lookup(tcp_hashinfo, skb, iph->saddr, src_port, iph->daddr, dst_port);
}

struct sock *__inet_lookup(tcp_hashinfo, skb, src_addr, src_port, dst_addr, dst_port)
{

	/* Convert dest_port# from network to host byte order */
	u16 hnum = ntohs(dst_port);
	
	/* First look for an established socket ... */
	sk = __inet_lookup_established(tcp_hashinfo, src_addr, src_port, dst_addr, hnum);
	
	if (sk)
		return sk;

	/* failing which, look for a LISTEN socket */
	return __inet_lookup_listener(tcp_hashinfo, skb, src_addr, src_port, dst_addr, hnum);
}

struct sock *__inet_lookup_listener(tcp_hashinfo, skb, src_addr, src_port, dst_addr, dst_port)
{
	/*
	* Use the destination port# to calculate a hash table slot# of the listen socket.
	* inet_lhashfn() returns a number between 0 and INET_LHTABLE_SIZE-1 (both
	* inclusive).
	*/
	unsigned int hash = inet_lhashfn(dst_port);
	
	/* Use this hash slot# to index the global LISTEN hash table */
	struct inet_listen_hashbucket *ilb = tcp_hashinfo->listening_hash[hash];
	
	/* Keep track of the best matching LISTEN socket thus far, and it’s “score” */
	struct sock *result = NULL, *sk;
	
	int hi_score = 0;
	for each socket, ‘sk’, in the selected hash bucket, ‘ilb’ {
	
		/*
		* Calculate the “score” of this LISTEN socket (sk) against the incoming skb.
		* Score is computed on some parameters, such as, exact destination port#,
		* destination IP address closest match (as against matching INADDR_ANY,
		* for example), with each criteria getting a different weight.
		*/
		score = compute_score(sk, dst_port, dst_addr);
		
		if (score > hi_score) 
		/* Highest score - best matched socket till now */
			if (sk->sk_reuseport) {
			/*
			* sk has SO_REUSEPORT feature enabled. Call inet_ehashfn()
			* with dest_addr, dest_port, src_addr and src_port to compute a
			* 2nd hash - phash.
			*/
			phash = inet_ehashfn(dst_addr, dst_port, src_addr, src_port);
			
			/* Select socket from sk’s SO_REUSEPORT group using phash.*/
			result = reuseport_select_sock(sk, phash);
			
			if (result)
				return result;
			}
			/* Update new best socket and it’s score */
			result = sk;
			hi_score = score;
		}
	}
	
	return result;
}

struct sock *reuseport_select_sock(struct sock *sk, unsigned int phash)
{
	/* Get control block of sockets in this SO_REUSEPORT group */
	struct sock_reuseport *reuse = sk->sk_reuseport_cb;
	
	/* Get count of sockets in the group */
	int num_socks = reuse->num_socks;
	
	/* Calculate value between 0 and 'num_socks-1' (both inclusive) */
	unsigned int index = reciprocal_scale(phash, num_socks);
	
	/* Index into the SO_REUSEPORT group using this index, and return that socket */
	return reuse->socks[index];
}

struct sock_reuseport {
	u16 max_socks; /* Allocated size of socks[] array */
	u16 num_socks; /* #Elements in socks[] */
	struct sock *socks[0]; /* All sockets added to this group */
};

static int inet_reuseport_add_sock(struct sock *new_sk)
{
	/* First check if another identical LISTEN socket, prev_sk,
	
	* exists. ... Then do the following:
	
	*/
	
	if (prev_sk) {
		/*
		
		* Not the first listener - do the following:
		* - Grow prev_sk->sk_reuseport_cb structure if required.
		* - Save new_sk socket pointer in prev_sk's socks[].
		* - prev_sk->sk_reuseport_cb->socks[num_socks] = new_sk;
		* - prev_sk->sk_reuseport_cb->num_socks++;
		* - Pointer assignment of the control block:
		* new_sk->sk_reuseport_cb = prev_sk->sk_reuseport_cb;
		*/
		
		return reuseport_add_sock(new_sk, prev_sk);
	}

	/*
	* First listener - do the following:
	* - allocate new_sk->sk_reuseport_cb to contain 128 socks[]
	* - new_sk->sk_reuseport_cb->max_socks = 128;
	* - new_sk->sk_reuseport_cb->socks[0] = new_sk;
	* - new_sk->sk_reuseport_cb->numsocks = 1;
	*/
	
	return reuseport_alloc(new_sk);
}

unsigned int index = reciprocal_scale(phash, num_socks);

return reuse->socks[index];

phash = inet_ehashfn(dst_addr, dst_port, src_addr, src_port);_

$ ss -tlnpe | grep :45000LISTEN 0 128 *:45000 *:* users:(("my_server",pid=3020,fd=3),("my_server",pid=3019,fd=3)) ino:3854904087 sk:37d5a0

$ ss -tlnpe | grep :45000

LISTEN 0 128 *:45000 *:* users:(("my_server",pid=1975,fd=3)) ino:3854935788 sk:37d59c

LISTEN 0 128 *:45000 *:* users:(("my_server",pid=1974,fd=3)) ino:3854935786 sk:37d59d

struct sock *inet_csk_accept(struct sock *sk)
{
	struct sock *newsk = NULL; /* client socket */
	
	/* Make sure that this socket is listening, and that it has something pending. */
	lock_sock(sk);
	if (sk->sk_state == TCP_LISTEN)
		if ("there are completed connections waiting to be accepted")
			newsk = get_first_connection(sk);
	release_sock(sk);
	return newsk;
}

With fork of the LISTEN socket
server-system-$ ./my_server 45000 48 0 # 0 indicates fork() of a LISTEN socket
client-system-$ time ./my_client <server-ip> 45000 48 1000000
real 4m45.471s
With SO_REUSEPORT
server-system-$ ./my_server 45000 48 1 # 1 indicates SO_REUSEPORT
client-system-$ time ./my_client <server-ip> 45000 48 1000000
real 1m36.766s

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/wait.h>
#include <netdb.h>

void create_children(int nprocs, int parent_pid)
{
	while (nprocs-- > 0) {
		if (getpid() == parent_pid && fork() < 0)
			exit(1);
	}
}

int main(int argc, char *argv[])
{
	int reuse_port, fd, cfd, nprocs, opt = 1, parent_pid = getpid();
	struct sockaddr_in server;
	
	if (argc != 4) {
		fprintf(stderr, "Port# #Procs {0->fork, or 1->SO_REUSEPORT}\n");
		return 1;
	}

	nprocs = atoi(argv[2]);
	reuse_port = atoi(argv[3]);
	
	if (reuse_port) /* proper SO_REUSEPORT */
		create_children(nprocs, parent_pid);
	
	if ((fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
		perror("socket");
		return 1;
	}

	if (reuse_port)
		setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, (char *)&opt, sizeof opt);
	
	server.sin_family = AF_INET;
	server.sin_addr.s_addr = INADDR_ANY;
	server.sin_port = htons(atoi(argv[1]));
	
	if (bind(fd, (struct sockaddr *)&server, sizeof server) < 0) {
		perror("bind");
		return 1;
	}
	
	if (!reuse_port) /* simple fork instead of SO_REUSEPORT */
		create_children(nprocs, parent_pid);
		
	if (parent_pid == getpid()) {
		while (wait(NULL) != -1); /* wait for all children */
	} else {
		listen(fd, SOMAXCONN);
		while (1) {
			if ((cfd = accept(fd, NULL, NULL)) < 0) {
				perror("accept");
				return 1;
			}
		close(cfd);
		}
	}
	
	return 0;
}

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <strings.h>
#include <sys/wait.h>
#include <netdb.h>

void create_children(int nprocs, int parent_pid)
{
	while (nprocs-- > 0) {
		if (getpid() == parent_pid && fork() < 0)
			exit(1);
	}
}

int main(int argc, char *argv[])
{
	int fd, count, nprocs, parent_pid = getpid();
	struct sockaddr_in server;
	struct hostent *server_ent;
	const struct linger nolinger = { .l_onoff = 1, .l_linger = 0 };
	
	if (argc != 5) {
		fprintf(stderr, "Server-IP Port# #Processes #Conns_per_Proc\n");
		return 1;
	}
	
	nprocs = atoi(argv[3]);
	count = atoi(argv[4]);
	
	if ((server_ent = gethostbyname(argv[1])) == NULL) {
		perror("gethostbyname");
		return 1;
	}
	
	bzero((char *)&server, sizeof server);
	server.sin_family = AF_INET;
	server.sin_port = htons(atoi(argv[2]));
	bcopy((char *)server_ent->h_addr, (char *)&server.sin_addr.s_addr, server_ent->h_length);
	
	create_children(nprocs, parent_pid);
	
	if (getpid() == parent_pid) {
		/* Parent does nothing other than wait for children */
		while (wait(NULL) != -1);
	} else {
		/* while the children connect() ‘count’ times to the server */
		while (count-- > 0) {
			if ((fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
				perror("socket");
				return 1;
			}
			
			if (connect(fd, (struct sockaddr *)&server, sizeof server) < 0) {
				perror("connect");
				return 1;
			}
			
			/* Reset connection to avoid TIME-WAIT state */
			setsockopt(fd, SOL_SOCKET, SO_LINGER, &nolinger, sizeof nolinger);
			close(fd);
		}
	}
	
	return 0;
}