From 252499c1513c45764d039af8732cd97b37c8c494 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl@samba.org>
Date: Thu, 9 Feb 2017 15:40:39 +0100
Subject: [PATCH 1/3] smbd: Streamline get_ea_names_from_file

Signed-off-by: Volker Lendecke <vl@samba.org>
Reviewed-by: Ralph Boehme <slow@samba.org>
Backported-by: Andreas Schneider <asn@samba.org>
Backported-from: 27daed8fcf95eed2df112dc1c30c3a40b5c9565b
---
 source3/smbd/trans2.c | 89 +++++++++++++++++++++++++++++----------------------
 1 file changed, 51 insertions(+), 38 deletions(-)

diff --git a/source3/smbd/trans2.c b/source3/smbd/trans2.c
index 98fd2af..49cfe9f 100644
--- a/source3/smbd/trans2.c
+++ b/source3/smbd/trans2.c
@@ -201,12 +201,14 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
 				files_struct *fsp, const char *fname,
 				char ***pnames, size_t *pnum_names)
 {
+	char smallbuf[1024];
 	/* Get a list of all xattrs. Max namesize is 64k. */
 	size_t ea_namelist_size = 1024;
-	char *ea_namelist = NULL;
+	char *ea_namelist = smallbuf;
+	char *to_free = NULL;
 
 	char *p;
-	char **names, **tmp;
+	char **names;
 	size_t num_names;
 	ssize_t sizeret = -1;
 	NTSTATUS status;
@@ -228,25 +230,24 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
 		return NT_STATUS_OK;
 	}
 
-	/*
-	 * TALLOC the result early to get the talloc hierarchy right.
-	 */
-
-	names = TALLOC_ARRAY(mem_ctx, char *, 1);
-	if (names == NULL) {
-		DEBUG(0, ("talloc failed\n"));
-		return NT_STATUS_NO_MEMORY;
+	if (fsp && fsp->fh->fd != -1) {
+		sizeret = SMB_VFS_FLISTXATTR(fsp, ea_namelist,
+					     ea_namelist_size);
+	} else {
+		sizeret = SMB_VFS_LISTXATTR(conn,
+					    fname,
+					    ea_namelist,
+					    ea_namelist_size);
 	}
 
-	while (ea_namelist_size <= 65536) {
-
-		ea_namelist = TALLOC_REALLOC_ARRAY(
-			names, ea_namelist, char, ea_namelist_size);
+	if ((sizeret == -1) && (errno == ERANGE)) {
+		ea_namelist_size = 65536;
+		ea_namelist = TALLOC_ARRAY(mem_ctx, char, ea_namelist_size);
 		if (ea_namelist == NULL) {
 			DEBUG(0, ("talloc failed\n"));
-			TALLOC_FREE(names);
 			return NT_STATUS_NO_MEMORY;
 		}
+		to_free = ea_namelist;
 
 		if (fsp && fsp->fh->fd != -1) {
 			sizeret = SMB_VFS_FLISTXATTR(fsp, ea_namelist,
@@ -255,25 +256,18 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
 			sizeret = SMB_VFS_LISTXATTR(conn, fname, ea_namelist,
 						    ea_namelist_size);
 		}
-
-		if ((sizeret == -1) && (errno == ERANGE)) {
-			ea_namelist_size *= 2;
-		}
-		else {
-			break;
-		}
 	}
 
 	if (sizeret == -1) {
-		TALLOC_FREE(names);
-		return map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix(errno);
+		TALLOC_FREE(to_free);
+		return status;
 	}
 
-	DEBUG(10, ("get_ea_list_from_file: ea_namelist size = %u\n",
-		   (unsigned int)sizeret));
+	DEBUG(10, ("ea_namelist size = %zd\n", sizeret));
 
 	if (sizeret == 0) {
-		TALLOC_FREE(names);
+		TALLOC_FREE(to_free);
 		return NT_STATUS_OK;
 	}
 
@@ -282,7 +276,7 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
 	 */
 
 	if (ea_namelist[sizeret-1] != '\0') {
-		TALLOC_FREE(names);
+		TALLOC_FREE(to_free);
 		return NT_STATUS_INTERNAL_ERROR;
 	}
 
@@ -295,26 +289,45 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
 		num_names += 1;
 	}
 
-	tmp = TALLOC_REALLOC_ARRAY(mem_ctx, names, char *, num_names);
-	if (tmp == NULL) {
+	*pnum_names = num_names;
+
+	if (pnames == NULL) {
+		TALLOC_FREE(to_free);
+		return NT_STATUS_OK;
+	}
+
+	names = TALLOC_ARRAY(mem_ctx, char *, num_names);
+	if (names == NULL) {
 		DEBUG(0, ("talloc failed\n"));
-		TALLOC_FREE(names);
+		TALLOC_FREE(to_free);
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	names = tmp;
+	if (ea_namelist == smallbuf) {
+		ea_namelist = talloc_memdup(names, smallbuf, sizeret);
+		if (ea_namelist == NULL) {
+			TALLOC_FREE(names);
+			return NT_STATUS_NO_MEMORY;
+		}
+	} else {
+		talloc_steal(names, ea_namelist);
+
+		ea_namelist = talloc_realloc(names, ea_namelist, char,
+					     sizeret);
+		if (ea_namelist == NULL) {
+			TALLOC_FREE(names);
+			return NT_STATUS_NO_MEMORY;
+		}
+	}
+
 	num_names = 0;
 
 	for (p = ea_namelist; p - ea_namelist < sizeret; p += strlen(p)+1) {
 		names[num_names++] = p;
 	}
 
-	if (pnames) {
-		*pnames = names;
-	} else {
-		TALLOC_FREE(names);
-	}
-	*pnum_names = num_names;
+	*pnames = names;
+
 	return NT_STATUS_OK;
 }
 
-- 
2.9.3


From 17563ab22ad19b34e1d9a1d12b2594c4186718b6 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl@samba.org>
Date: Tue, 25 Oct 2016 12:28:12 +0200
Subject: [PATCH 2/3] lib/util/charset: Optimize next_codepoint for the ascii
 case

Signed-off-by: Volker Lendecke <vl@samba.org>
Reviewed-by: Ralph Boehme <slow@samba.org>

(cherry picked from commit 07d9a909ba6853fb0b96f6d86e4cf0d5d1b35b28)
---
 lib/util/charset/codepoints.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/util/charset/codepoints.c b/lib/util/charset/codepoints.c
index 5ee95a8..7d157a3 100644
--- a/lib/util/charset/codepoints.c
+++ b/lib/util/charset/codepoints.c
@@ -495,6 +495,10 @@ _PUBLIC_ codepoint_t next_codepoint_ext(const char *str, charset_t src_charset,
 
 _PUBLIC_ codepoint_t next_codepoint(const char *str, size_t *size)
 {
+	if ((str[0] & 0x80) == 0) {
+		*size = 1;
+		return str[0];
+	}
 	return next_codepoint_convenience(get_iconv_convenience(), str, size);
 }
 
-- 
2.9.3


From ac8f6faa891fd282fb39ccb8e75a364bf97a5f2b Mon Sep 17 00:00:00 2001
From: Andreas Schneider <asn@samba.org>
Date: Thu, 9 Feb 2017 15:05:01 +0100
Subject: [PATCH 3/3] s3-vfs: Only walk the directory once in
 open_and_sort_dir()

On a slow filesystem or network filesystem this can make a huge
difference.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=12571

Signed-off-by: Andreas Schneider <asn@samba.org>
---
 source3/modules/vfs_dirsort.c | 61 +++++++++++++++++++++++++++----------------
 1 file changed, 38 insertions(+), 23 deletions(-)

diff --git a/source3/modules/vfs_dirsort.c b/source3/modules/vfs_dirsort.c
index 698e96b..66582e6 100644
--- a/source3/modules/vfs_dirsort.c
+++ b/source3/modules/vfs_dirsort.c
@@ -68,8 +68,10 @@ static bool get_sorted_dir_mtime(vfs_handle_struct *handle,
 static bool open_and_sort_dir(vfs_handle_struct *handle,
 				struct dirsort_privates *data)
 {
-	unsigned int i = 0;
-	unsigned int total_count = 0;
+	uint32_t total_count = 0;
+	/* This should be enough for most use cases */
+	uint32_t dirent_allocated = 64;
+	SMB_STRUCT_DIRENT *dp;
 
 	data->number_of_entries = 0;
 
@@ -77,38 +79,51 @@ static bool open_and_sort_dir(vfs_handle_struct *handle,
 		return false;
 	}
 
-	while (SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL)
-	       != NULL) {
-		total_count++;
-	}
-
-	if (total_count == 0) {
+	dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL);
+	if (dp == NULL) {
 		return false;
 	}
 
-	/* Open the underlying directory and count the number of entries
-	   Skip back to the beginning as we'll read it again */
-	SMB_VFS_NEXT_REWINDDIR(handle, data->source_directory);
-
 	/* Set up an array and read the directory entries into it */
 	TALLOC_FREE(data->directory_list); /* destroy previous cache if needed */
 	data->directory_list = talloc_zero_array(data,
 						 SMB_STRUCT_DIRENT,
-						 total_count);
-	if (!data->directory_list) {
+						 dirent_allocated);
+	if (data->directory_list == NULL) {
 		return false;
 	}
-	for (i = 0; i < total_count; i++) {
-		SMB_STRUCT_DIRENT *dp = SMB_VFS_NEXT_READDIR(handle,
-						data->source_directory,
-						NULL);
-		if (dp == NULL) {
-			break;
+
+	do {
+		if (total_count >= dirent_allocated) {
+			struct dirent *dlist;
+
+			/*
+			 * Be memory friendly.
+			 *
+			 * We should not double the amount of memory. With a lot
+			 * of files we reach easily 50MB, and doubling will
+			 * get much bigger just for a few files more.
+			 *
+			 * For 200k files this means 50 memory reallocations.
+			 */
+			dirent_allocated += 4096;
+
+			dlist = talloc_realloc(data,
+					       data->directory_list,
+					       SMB_STRUCT_DIRENT,
+					       dirent_allocated);
+			if (dlist == NULL) {
+				break;
+			}
+			data->directory_list = dlist;
 		}
-		data->directory_list[i] = *dp;
-	}
+		data->directory_list[total_count] = *dp;
+
+		total_count++;
+		dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL);
+	} while (dp != NULL);
 
-	data->number_of_entries = i;
+	data->number_of_entries = total_count;
 
 	/* Sort the directory entries by name */
 	TYPESAFE_QSORT(data->directory_list, data->number_of_entries, compare_dirent);
-- 
2.9.3

