@@ -1,3 +1,12 @@

+Mon Feb 16 20:36:01 EET 2009 (edwin)

+------------------------------------

+ * clamd/, libclamav/, shared/, unit_tests/: introduce timeouts for

+ send(), this is needed for IDSESSION to work reliably, since a buggy

+ client implementation may get stuck in send(), and then clamd gets

+ stuck in send() -> deadlock. To avoid this we need nonblocking

+ sockets, and (low) timeouts on send.  Add more tests for clamd protocol,

+ including a stress test for IDSESSION.

+

 Mon Feb 16 16:15:21 EET 2009 (edwin)

 ------------------------------------

  * unit_tests/check_clamd.c: test for FILDES

@@ -190,7 +190,8 @@ int scan_callback(struct stat *sb, char *filename, const char *msg, enum cli_ftw

     }

     if (access(filename, R_OK)) {

-	conn_reply(scandata->conn, filename, "Access denied.", "ERROR");

+	if (conn_reply(scandata->conn, filename, "Access denied.", "ERROR") == -1)

+	    return CL_ETIMEOUT;

 	logg("*Access denied: %s\n", filename);

 	scandata->errors++;

 	free(filename);

@@ -202,14 +203,21 @@ int scan_callback(struct stat *sb, char *filename, const char *msg, enum cli_ftw

     ret = cl_scanfile(filename, &virname, &scandata->scanned, scandata->engine, scandata->options);

     thrmgr_setactivetask(NULL, NULL);

+    if (thrmgr_group_need_terminate(scandata->conn->group)) {

+	logg("*Client disconnected while scanjob was active\n");

+	return ret == CL_ETIMEOUT ? ret : CL_BREAK;

+    }

+

     if (ret == CL_VIRUS) {

 	scandata->infected++;

-	conn_reply(scandata->conn, filename, virname, "FOUND");

+	if (conn_reply(scandata->conn, filename, virname, "FOUND") == -1)

+	    return CL_ETIMEOUT;

 	logg("~%s: %s FOUND\n", filename, virname);

 	virusaction(filename, virname, scandata->opts);

     } else if (ret != CL_CLEAN) {

 	scandata->errors++;

-	conn_reply(scandata->conn, filename, cl_strerror(ret), "ERROR");

+	if (conn_reply(scandata->conn, filename, cl_strerror(ret), "ERROR") == -1)

+	    return CL_ETIMEOUT;

 	logg("~%s: %s ERROR\n", filename, cl_strerror(ret));

     } else if (logok) {

 	logg("~%s: OK\n", filename);

@@ -219,10 +227,6 @@ int scan_callback(struct stat *sb, char *filename, const char *msg, enum cli_ftw

     if(ret == CL_EMEM) /* stop scanning */

 	return ret;

-    if (thrmgr_group_need_terminate(scandata->conn->group)) {

-	logg("^Client disconnected while scanjob was active\n");

-	return CL_BREAK;

-    }

     if (type == TYPE_SCAN) {

 	/* virus -> break */

 	return ret;

@@ -246,8 +250,9 @@ int scanfd(const int fd, const client_conn_t *conn, unsigned long int *scanned,

 	else

 	    snprintf(fdstr, sizeof(fdstr), "fd[%d]", fd);

 	if(fstat(fd, &statbuf) == -1 || !S_ISREG(statbuf.st_mode)) {

-		conn_reply(conn, fdstr, "Not a regular file", "ERROR");

 		logg("%s: Not a regular file. ERROR\n", fdstr);

+		if (conn_reply(conn, fdstr, "Not a regular file", "ERROR") == -1)

+		    return CL_ETIMEOUT;

 		return -1;

 	}

@@ -255,15 +260,23 @@ int scanfd(const int fd, const client_conn_t *conn, unsigned long int *scanned,

 	ret = cl_scandesc(fd, &virname, scanned, engine, options);

 	thrmgr_setactivetask(NULL, NULL);

+	if (thrmgr_group_need_terminate(conn->group)) {

+	    logg("*Client disconnected while scanjob was active\n");

+	    return ret == CL_ETIMEOUT ? ret : CL_BREAK;

+	}

+

 	if(ret == CL_VIRUS) {

-		conn_reply(conn, fdstr, virname, "FOUND");

+		if (conn_reply(conn, fdstr, virname, "FOUND") == -1)

+		    ret = CL_ETIMEOUT;

 		logg("%s: %s FOUND\n", fdstr, virname);

 		virusaction(fdstr, virname, opts);

 	} else if(ret != CL_CLEAN) {

-		conn_reply(conn, fdstr, cl_strerror(ret), "ERROR");

+		if (conn_reply(conn, fdstr, cl_strerror(ret), "ERROR") == -1)

+		    ret = CL_ETIMEOUT;

 		logg("%s: %s ERROR\n", fdstr, cl_strerror(ret));

 	} else {

-		conn_reply_single(conn, fdstr, "OK");

+		if (conn_reply_single(conn, fdstr, "OK") == CL_ETIMEOUT)

+		    ret = CL_ETIMEOUT;

 		if(logok)

 			logg("%s: OK\n", fdstr);

 	}

@@ -43,6 +43,7 @@

 #include <unistd.h>

 #endif

+#include <fcntl.h>

 #include <arpa/inet.h>

 #include "libclamav/clamav.h"

@@ -358,7 +359,20 @@ static void *acceptloop_th(void *arg)

 	    pthread_mutex_unlock(&exit_mutex);

 	    if (new_sd >= 0) {

-		int ret;

+		int ret, flags;

+

+#ifdef F_GETFL

+		flags = fcntl(new_sd, F_GETFL, 0);

+		if (flags != -1) {

+		    if (fcntl(new_sd, F_SETFL, flags | O_NONBLOCK) == -1) {

+			logg("^Can't set socket to nonblocking mode, errno %d\n",

+			     errno);

+		    }

+		} else {

+			logg("^Can't get socket flags, errno %d\n", errno);

+		}

+#endif

+

 		pthread_mutex_lock(&recv_fds->buf_mutex);

 		ret = fds_add(recv_fds, new_sd, 0);

 		pthread_mutex_unlock(&recv_fds->buf_mutex);

@@ -882,6 +896,11 @@ int recvloop_th(int *socketds, unsigned nsockets, struct cl_engine *engine, unsi

 			}

 			error = 1;

 		    }

+		    if (thrmgr_group_need_terminate(conn.group)) {

+			logg("*RECVTH: have to terminate group\n");

+			error = CL_ETIMEOUT;

+			break;

+		    }

 		    if (error || !conn.group || rc) {

 			if (rc && thrmgr_group_finished(conn.group, EXIT_OK)) {

 			    logg("*RECVTH: closing conn, group finished\n");

@@ -1023,7 +1042,7 @@ int recvloop_th(int *socketds, unsigned nsockets, struct cl_engine *engine, unsi

 			buf->off = 0;

 		    }

 		}

-		if (error) {

+		if (error && error != CL_ETIMEOUT) {

 		    conn_reply_error(&conn, "Error processing command.");

 		}

 	    }

@@ -1036,7 +1055,8 @@ int recvloop_th(int *socketds, unsigned nsockets, struct cl_engine *engine, unsi

 		    }

 		    buf->dumpfd = -1;

 		}

-		if (thrmgr_group_terminate(buf->group)) {

+		thrmgr_group_terminate(buf->group);

+		if (thrmgr_group_finished(buf->group, EXIT_ERROR)) {

 		    logg("*RECVTH: shutting down socket after error\n");

 		    shutdown(buf->fd, 2);

 		    closesocket(buf->fd);

@@ -1055,7 +1075,8 @@ int recvloop_th(int *socketds, unsigned nsockets, struct cl_engine *engine, unsi

 	    for (i=0;i < fds->nfds; i++) {

 		if (fds->buf[i].fd == -1)

 		    continue;

-		if (thrmgr_group_terminate(fds->buf[i].group)) {

+		thrmgr_group_terminate(fds->buf[i].group);

+		if (thrmgr_group_finished(fds->buf[i].group, EXIT_ERROR)) {

 		    shutdown(fds->buf[i].fd, 2);

 		    closesocket(fds->buf[i].fd);

 		    fds->buf[i].fd = -1;

@@ -183,7 +183,7 @@ int command(client_conn_t *conn, int *virus)

     jobgroup_t *group = NULL;

     if (thrmgr_group_need_terminate(conn->group)) {

-	logg("^Client disconnected while command was active\n");

+	logg("*Client disconnected while command was active\n");

 	if (conn->scanfd != -1)

 	    close(conn->scanfd);

 	return 1;

@@ -235,11 +235,14 @@ int command(client_conn_t *conn, int *virus)

 		conn_reply_error(conn, "FILDES: didn't receive file descriptor.");

 	    else {

 		ret = scanfd(conn->scanfd, conn, NULL, engine, options, opts, desc, 0);

-		if (ret == CL_VIRUS)

+		if (ret == CL_VIRUS) {

 		    *virus = 1;

-		if (ret == CL_EMEM) {

+		} else if (ret == CL_EMEM) {

 		    if(optget(opts, "ExitOnOOM")->enabled)

 			ret = -1;

+		} else if (ret == CL_ETIMEOUT) {

+			thrmgr_group_terminate(conn->group);

+			ret = 1;

 		} else

 		    ret = 0;

 	    }

@@ -269,11 +272,14 @@ int command(client_conn_t *conn, int *virus)

 	    return 0;

 	case COMMAND_INSTREAMSCAN:

 	    ret = scanfd(conn->scanfd, conn, NULL, engine, options, opts, desc, 1);

-	    if (ret == CL_VIRUS)

+	    if (ret == CL_VIRUS) {

 		*virus = 1;

-	    if (ret == CL_EMEM) {

+	    } else if (ret == CL_EMEM) {

 		if(optget(opts, "ExitOnOOM")->enabled)

 		    ret = -1;

+	    } else if (ret == CL_ETIMEOUT) {

+		thrmgr_group_terminate(conn->group);

+		ret = 1;

 	    } else

 		ret = 0;

 	    if (ftruncate(conn->scanfd, 0) == -1) {

@@ -292,7 +298,8 @@ int command(client_conn_t *conn, int *virus)

 	flags |= CLI_FTW_FOLLOW_DIR_SYMLINK;

     if (optget(opts, "FollowFileSymlinks")->enabled)

 	flags |= CLI_FTW_FOLLOW_FILE_SYMLINK;

-    if (cli_ftw(conn->filename, flags,  maxdirrec ? maxdirrec : INT_MAX, scan_callback, &data) == CL_EMEM) 

+    ret = cli_ftw(conn->filename, flags,  maxdirrec ? maxdirrec : INT_MAX, scan_callback, &data);

+    if (ret == CL_EMEM)

 	if(optget(opts, "ExitOnOOM")->enabled)

 	    return -1;

     if (scandata.group && conn->cmdtype == COMMAND_MULTISCAN) {

@@ -304,9 +311,13 @@ int command(client_conn_t *conn, int *virus)

     }

     if (ok + error == total && (error != total)) {

-	conn_reply_single(conn, conn->filename, "OK");

+	if (conn_reply_single(conn, conn->filename, "OK") == -1)

+	    ret = CL_ETIMEOUT;

     }

     *virus = total - (ok + error);

+

+    if (ret == CL_ETIMEOUT)

+	thrmgr_group_terminate(conn->group);

     return error;

 }

@@ -828,18 +828,13 @@ int thrmgr_group_need_terminate(jobgroup_t *group)

     return ret;

 }

-/* returns

- *  0 - flag set, jobs still active

- *  1 - last job exited */

-int thrmgr_group_terminate(jobgroup_t *group)

+void thrmgr_group_terminate(jobgroup_t *group)

 {

-    if (thrmgr_group_finished(group, EXIT_ERROR))

-	return 1;

-    /* we are not last active job, now

-     * the last active job will free resources */

-    pthread_mutex_lock(&group->mutex);

-    group->force_exit = 1;

-    pthread_mutex_unlock(&group->mutex);

-

-    return 0;

+    if (group) {

+	/* we may not be the last active job, now

+	 * the last active job will free resources */

+	pthread_mutex_lock(&group->mutex);

+	group->force_exit = 1;

+	pthread_mutex_unlock(&group->mutex);

+    }

 }

@@ -101,7 +101,7 @@ int thrmgr_group_dispatch(threadpool_t *threadpool, jobgroup_t *group, void *use

 void thrmgr_group_waitforall(jobgroup_t *group, unsigned *ok, unsigned *error, unsigned *total);

 int thrmgr_group_finished(jobgroup_t *group, enum thrmgr_exit exitc);

 int thrmgr_group_need_terminate(jobgroup_t *group);

-int thrmgr_group_terminate(jobgroup_t *group);

+void thrmgr_group_terminate(jobgroup_t *group);

 jobgroup_t *thrmgr_group_new(void);

 int thrmgr_printstats(int outfd);

 void thrmgr_setactivetask(const char *filename, const char* command);

@@ -57,6 +57,7 @@ typedef enum {

     CL_ETMPDIR,

     CL_EMAP,

     CL_EMEM,

+    CL_ETIMEOUT,

     /* internal (not reported outside libclamav) */

     CL_BREAK,

@@ -47,6 +47,10 @@

 #include <sys/types.h>

 #endif

+#ifdef HAVE_SYS_SELECT_H

+#include <sys/select.h>

+#endif

+

 #if defined(USE_SYSLOG) && !defined(C_AIX)

 #include <syslog.h>

 #endif

@@ -88,7 +92,7 @@ short logg_syslog;

 #endif

 short int mprintf_disabled = 0, mprintf_verbose = 0, mprintf_quiet = 0,

-	  mprintf_stdout = 0, mprintf_nowarn = 0;

+	  mprintf_stdout = 0, mprintf_nowarn = 0, mprintf_send_timeout = 100;

 #define ARGLEN(args, str, len)			    \

 {						    \

@@ -127,7 +131,7 @@ int mdprintf(int desc, const char *str, ...)

 {

 	va_list args;

 	char buffer[512], *abuffer = NULL, *buff;

-	int bytes, len, ret;

+	int bytes, todo, len, ret;

     ARGLEN(args, str, len);

@@ -156,12 +160,36 @@ int mdprintf(int desc, const char *str, ...)

     if((size_t) bytes >= len)

 	bytes = len - 1;

-    ret = send(desc, buff, bytes, 0);

+    todo = bytes;

+    while (todo > 0) {

+	ret = send(desc, buff, bytes, 0);

+	if (ret < 0) {

+	    struct timeval tv;

+	    if (errno != EWOULDBLOCK)

+		break;

+	    tv.tv_sec = 0;

+	    tv.tv_usec = mprintf_send_timeout*1000;

+	    do {

+		fd_set wfds;

+		FD_ZERO(&wfds);

+		FD_SET(desc, &wfds);

+		ret = select(desc+1, NULL, &wfds, NULL, &tv);

+	    } while (ret < 0 && errno == EINTR);

+	    if (!ret) {

+		/* timed out */

+		ret = -1;

+		break;

+	    }

+	} else {

+	    todo -= ret;

+	    buff += ret;

+	}

+    }

     if(len > sizeof(buffer))

 	free(abuffer);

-    return bytes;

+    return ret < 0 ? -1 : bytes;

 }

 void logg_close(void)

@@ -162,7 +162,7 @@ static void test_command(const char *cmd, size_t len, const char *extra, const c

     if (extra) {

 	rc = send(sockd, extra, strlen(extra), 0);

-	fail_unless_fmt((size_t)rc == strlen(extra), "Unable to send(): %s\n", strerror(errno));

+	fail_unless_fmt((size_t)rc == strlen(extra), "Unable to send() extra for %s: %s\n", cmd, strerror(errno));

     }

     recvdata = recvfull(sockd, &len);

@@ -215,11 +215,15 @@ START_TEST (test_compat_commands)

     test_command(nsend, strlen(nsend), test->extra, nreply, strlen(nreply));

     conn_teardown();

-    /* one packet, \r\n delimited command, followed by "extra" if needed */

-    snprintf(nsend, sizeof(nsend), "%s\r\n", test->command);

-    conn_setup();

-    test_command(nsend, strlen(nsend), test->extra, nreply, strlen(nreply));

-    conn_teardown();

+    if (!test->extra) {

+	/* FILDES won't support this, because it expects

+	 * strlen("FILDES\n") characters, then 1 character and the FD. */

+	/* one packet, \r\n delimited command, followed by "extra" if needed */

+	snprintf(nsend, sizeof(nsend), "%s\r\n", test->command);

+	conn_setup();

+	test_command(nsend, strlen(nsend), test->extra, nreply, strlen(nreply));

+	conn_teardown();

+    }

 }

 END_TEST

@@ -295,22 +299,28 @@ START_TEST (test_instream)

 }

 END_TEST

-static void tst_fildes(const char *cmd, size_t len, int fd,

-			const char *expect, size_t expect_len, int closefd)

+static int sendmsg_fd(int sockd, const char *mesg, size_t msg_len, int fd, int singlemsg)

 {

     struct msghdr msg;

     struct cmsghdr *cmsg;

     unsigned char fdbuf[CMSG_SPACE(sizeof(int))];

-    char dummy[] = "";

-    off_t pos;

+    char dummy[BUFSIZ];

     struct iovec iov[1];

-    char *recvdata, *p;

     int rc;

-    conn_setup();

-

-    iov[0].iov_base = dummy;

-    iov[0].iov_len = 1;

+    if (!singlemsg) {

+	/* send FILDES\n and then a single character + ancillary data */

+	dummy[0] = '\0';

+	iov[0].iov_base = dummy;

+	iov[0].iov_len = 1;

+    } else {

+	/* send single message with ancillary data */

+	fail_unless(msg_len < sizeof(dummy)-1);

+	memcpy(dummy, mesg, msg_len);

+	dummy[msg_len] = '\0';

+	iov[0].iov_base = dummy;

+	iov[0].iov_len = msg_len + 1;

+    }

     memset(&msg, 0, sizeof(msg));

     msg.msg_control = fdbuf;

@@ -324,10 +334,25 @@ static void tst_fildes(const char *cmd, size_t len, int fd,

     cmsg->cmsg_type = SCM_RIGHTS;

     *(int *)CMSG_DATA(cmsg) = fd;

-    fail_unless_fmt(write(sockd, cmd, len) == len, "Failed to write: %s\n", strerror(errno));

+    if (!singlemsg) {

+	rc = send(sockd, mesg, msg_len, 0);

+	if (rc == -1)

+	    return rc;

+    }

+

+    return sendmsg(sockd, &msg, 0);

+}

+

+static void tst_fildes(const char *cmd, size_t len, int fd,

+			const char *expect, size_t expect_len, int closefd, int singlemsg)

+{

+    off_t pos;

+    char *recvdata, *p;

+    int rc;

-    rc = sendmsg(sockd, &msg, 0);

-    fail_unless_fmt(rc != -1, "Failed to sendmsg: %s\n", strerror(errno));

+    conn_setup();

+    fail_unless_fmt(sendmsg_fd(sockd, cmd, len, fd, singlemsg) != -1,

+		     "Failed to sendmsg: %s\n", strerror(errno));

     if (closefd)

 	close(fd);

@@ -353,48 +378,110 @@ static void tst_fildes(const char *cmd, size_t len, int fd,

 #define FOUNDFDREPLY " ClamAV-Test-File.UNOFFICIAL FOUND"

 #define CLEANFDREPLY " OK"

+static struct cmds {

+    const char *cmd;

+    const char term;

+    const char *file;

+    const char *reply;

+} fildes_cmds[] =

+{

+    {"FILDES", '\n', SCANFILE, FOUNDFDREPLY},

+    {"nFILDES", '\n', SCANFILE, FOUNDFDREPLY},

+    {"zFILDES", '\0', SCANFILE, FOUNDFDREPLY},

+    {"FILDES", '\n', CLEANFILE, CLEANFDREPLY},

+    {"nFILDES", '\n', CLEANFILE, CLEANFDREPLY},

+    {"zFILDES", '\0', CLEANFILE, CLEANFDREPLY}

+};

+

 START_TEST (test_fildes)

 {

-    char nreply[BUFSIZ];

+    char nreply[BUFSIZ], nsend[BUFSIZ];

     int fd = open(SCANFILE, O_RDONLY);

+    int closefd;

+    int singlemsg;

+    size_t i;

+    const struct cmds *cmd;

+    size_t nreply_len, nsend_len;

+

+    switch (_i&3) {

+	case 0:

+	    closefd = 0;

+	    singlemsg = 0;

+	    break;

+	case 1:

+	    closefd = 1;

+	    singlemsg = 0;

+	    break;

+	case 2:

+	    closefd = 0;

+	    singlemsg = 1;

+	    break;

+	case 3:

+	    closefd = 1;

+	    singlemsg = 1;

+	    break;

+    }

-    snprintf(nreply, sizeof(nreply), "%s\n", FOUNDFDREPLY);

+    cmd = &fildes_cmds[_i/4];

+    nreply_len = snprintf(nreply, sizeof(nreply), "%s%c", cmd->reply, cmd->term);

+    nsend_len = snprintf(nsend, sizeof(nsend), "%s%c", cmd->cmd, cmd->term);

+    fd = open(cmd->file, O_RDONLY);

     fail_unless_fmt(fd != -1, "Failed to open: %s\n", strerror(errno));

-    tst_fildes("FILDES\n", strlen("FILDES\n"), fd, nreply, strlen(nreply), 0);

-    tst_fildes("nFILDES\n", strlen("nFILDES\n"), fd, nreply, strlen(nreply), 0);

-    tst_fildes("zFILDES", sizeof("zFILDES"), fd, FOUNDFDREPLY, sizeof(FOUNDFDREPLY), 0);

+    tst_fildes(nsend, nsend_len, fd, nreply, nreply_len, closefd, singlemsg);

-    close(fd);

+    if (!closefd) {

+	/* closefd: 

+	 *  1 - close fd right after sending

+	 *  0 - close fd after receiving reply */

+	close(fd);

+    }

+}

+END_TEST

-    snprintf(nreply, sizeof(nreply), "%s\n", CLEANFDREPLY);

+START_TEST (test_fildes_many)

+{

+    const char idsession[] = "zIDSESSION";

+    int dummyfd, dummycleanfd, i, killed = 0;

-    fd = open(CLEANFILE, O_RDONLY);

-    fail_unless_fmt(fd != -1, "Failed to open: %s\n", strerror(errno));

+    conn_setup();

+    dummyfd = open(SCANFILE, O_RDONLY);

+    fail_unless_fmt(dummyfd != -1, "failed to open %s: %s\n", SCANFILE, strerror(errno));

+

+    fail_unless_fmt(send(sockd, idsession, sizeof(idsession), 0) == sizeof(idsession), "send IDSESSION failed\n");

+    for (i=0; i < 2048; i++) {

+	if (sendmsg_fd(sockd, "zFILDES", sizeof("zFILDES"), dummyfd, 1) == -1) {

+	    killed = 1;

+	    break;

+	}

+    }

-    tst_fildes("FILDES\n", strlen("FILDES\n"), fd, nreply, strlen(nreply), 0);

-    tst_fildes("nFILDES\n", strlen("nFILDES\n"), fd, nreply, strlen(nreply), 0);

-    tst_fildes("zFILDES", sizeof("zFILDES"), fd, CLEANFDREPLY, sizeof(CLEANFDREPLY), 0);

+    fail_unless(killed, "Clamd did not kill connection when overloaded!\n");

-    snprintf(nreply, sizeof(nreply), "%s\n", FOUNDFDREPLY);

-    fd = open(SCANFILE, O_RDONLY);

-    fail_unless_fmt(fd != -1, "Failed to open: %s\n", strerror(errno));

-    tst_fildes("FILDES\n", strlen("FILDES\n"), fd, nreply, strlen(nreply), 1);

+    close(dummyfd);

+    conn_teardown();

 }

 END_TEST

 static Suite *test_clamd_suite(void)

 {

     Suite *s = suite_create("clamd");

-    TCase *tc_commands = tcase_create("clamd commands");

+    TCase *tc_commands, *tc_stress;

+

+    tc_commands = tcase_create("clamd commands");

     suite_add_tcase(s, tc_commands);

     tcase_add_unchecked_fixture(tc_commands, commands_setup, commands_teardown);

     tcase_add_loop_test(tc_commands, test_basic_commands, 0, sizeof(basic_tests)/sizeof(basic_tests[0]));

     tcase_add_loop_test(tc_commands, test_compat_commands, 0, sizeof(basic_tests)/sizeof(basic_tests[0]));

-    tcase_add_test(tc_commands, test_fildes);

+    tcase_add_loop_test(tc_commands, test_fildes, 0, 4*sizeof(fildes_cmds)/sizeof(fildes_cmds[0]));

     tcase_add_test(tc_commands, test_stats);

     tcase_add_test(tc_commands, test_instream);

+

+    tc_stress = tcase_create("clamd stress test");

+    suite_add_tcase(s, tc_stress);

+    tcase_add_test(tc_stress, test_fildes_many);

+

     return s;

 }