rpms
/
nbdkit
20
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

merge into: rpms:i10c-beta
Branches Tags
rpms:i10c-beta
rpms:c10-beta
rpms:i10cs
rpms:cs10
rpms:i9c-beta
rpms:c9-beta
rpms:i8c-beta-stream-rhel
rpms:c8-beta-stream-rhel
rpms:i8c-stream-rhel
rpms:c8-stream-rhel
rpms:i9c
rpms:c9
...
pull from: rpms:c9
Branches Tags
rpms:i10c-beta
rpms:c10-beta
rpms:i10cs
rpms:cs10
rpms:i9c-beta
rpms:c9-beta
rpms:i8c-beta-stream-rhel
rpms:c8-beta-stream-rhel
rpms:i8c-stream-rhel
rpms:c8-stream-rhel
rpms:i9c
rpms:c9

No commits in common. 'i10c-beta' and 'c9' have entirely different histories.
i10c-beta ... c9

40 changed files with 5444 additions and 1855 deletions
Show Stats

2
.gitignore vendored
Unescape View File

@ -1,2 +1,2 @@
SOURCES/libguestfs.keyring
SOURCES/nbdkit-1.40.0.tar.gz
SOURCES/nbdkit-1.34.2.tar.gz

2
.nbdkit.metadata
Unescape View File

@ -1,2 +1,2 @@
cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring
dce2a6598adce9a5362622756041349926cef380 SOURCES/nbdkit-1.40.0.tar.gz
d1c190dcfe1e601e0eeb4862c49df45c7052955b SOURCES/nbdkit-1.34.2.tar.gz

51
SOURCES/0001-server-crypto.c-Fix-fallback-functions-used-when-gnu.patch
Unescape View File

@ -1,51 +0,0 @@
From 7563596fdb7587dfb8708b5eeeb4b97d738ba79d Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 24 Jul 2024 11:22:28 +0100
Subject: [PATCH] server/crypto.c: Fix fallback functions used when gnutls is
unavailable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The return value was wrong resulting in these errors:
crypto.c: In function ‘nbdkit_peer_tls_dn’:
crypto.c:945:3: error: return makes pointer from integer without a cast [-Werror]
return -1;
^
crypto.c: In function ‘nbdkit_peer_tls_issuer_dn’:
crypto.c:953:3: error: return makes pointer from integer without a cast [-Werror]
return -1;
^
Fixes: commit a3759199c93300cf9de24a129c4141609dacb047
Fixes: commit bf77f76112e3ffbc9149a54fbeb0a505310f28a2
---
server/crypto.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/server/crypto.c b/server/crypto.c
index abdc4801..fa408c75 100644
--- a/server/crypto.c
+++ b/server/crypto.c
@@ -942,7 +942,7 @@ nbdkit_peer_tls_dn (void)
{
nbdkit_error ("%s is not supported on this platform",
"nbdkit_peer_tls_dn");
- return -1;
+ return NULL;
}
NBDKIT_DLL_PUBLIC char *
@@ -950,7 +950,7 @@ nbdkit_peer_tls_issuer_dn (void)
{
nbdkit_error ("%s is not supported on this platform",
"nbdkit_peer_tls_issuer_dn");
- return -1;
+ return NULL;
}
#endif /* !HAVE_GNUTLS */
--
2.43.0

33
SOURCES/0001-tests-test-connect.c-Skip-if-exit-with-parent-is-not.patch
Unescape View File

@ -0,0 +1,33 @@
From 41775572a5beb8ae49287271af31264684b2bbe3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 21 Jul 2023 19:00:31 +0100
Subject: [PATCH] tests/test-connect.c: Skip if --exit-with-parent is not
supported
Fixes: commit 933d7401ff623077ba43b4ef1e7f16a7864c5fde
(cherry picked from commit 59664b8b146edcba0353de628528e0a2035b3ba7)
---
tests/test-connect.c | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/tests/test-connect.c b/tests/test-connect.c
index f463415b..d7118330 100644
--- a/tests/test-connect.c
+++ b/tests/test-connect.c
@@ -47,6 +47,13 @@ main (int argc, char *argv[])
struct nbd_handle *nbd;
int64_t size;
+ if (system ("nbdkit --exit-with-parent --version") != 0) {
+ printf ("%s: --exit-with-parent is not implemented on this platform, "
+ "skipping\n",
+ argv[0]);
+ exit (77);
+ }
+
nbd = nbd_create ();
if (nbd == NULL) {
fprintf (stderr, "%s\n", nbd_get_error ());
--
2.39.3

149
SOURCES/0002-server-log-Move-preserve-errno-to-log_verror-functio.patch
Unescape View File

@ -1,149 +0,0 @@
From f2c644d4495d5e75883ff729936102c90489e8d8 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 23 Jul 2024 14:46:41 +0100
Subject: [PATCH] server: log: Move preserve errno to log_verror function
This neutral code refactoring just moves the place where we preserve
errno out one layer, but should have no other effect.
---
server/internal.h | 8 ++++----
server/log-stderr.c | 9 ++-------
server/log-syslog.c | 13 ++++---------
server/log.c | 12 ++++++++----
4 files changed, 18 insertions(+), 24 deletions(-)
diff --git a/server/internal.h b/server/internal.h
index 0b0507e4..1a783e3f 100644
--- a/server/internal.h
+++ b/server/internal.h
@@ -340,10 +340,10 @@ extern void free_debug_flags (void);
extern void log_verror (const char *fs, va_list args);
/* log-*.c */
-extern void log_stderr_verror (const char *fs, va_list args)
- ATTRIBUTE_FORMAT_PRINTF (1, 0);
-extern void log_syslog_verror (const char *fs, va_list args)
- ATTRIBUTE_FORMAT_PRINTF (1, 0);
+extern void log_stderr_verror (int orig_errno, const char *fs, va_list args)
+ ATTRIBUTE_FORMAT_PRINTF (2, 0);
+extern void log_syslog_verror (int orig_errno, const char *fs, va_list args)
+ ATTRIBUTE_FORMAT_PRINTF (2, 0);
/* vfprintf.c */
#if !HAVE_VFPRINTF_PERCENT_M
diff --git a/server/log-stderr.c b/server/log-stderr.c
index 8a55f5df..4d8b09da 100644
--- a/server/log-stderr.c
+++ b/server/log-stderr.c
@@ -43,12 +43,9 @@
#include "internal.h"
-/* Note: preserves the previous value of errno. */
void
-log_stderr_verror (const char *fs, va_list args)
+log_stderr_verror (int orig_errno, const char *fs, va_list args)
{
- int err = errno; /* must be first line of function */
-
const char *name = threadlocal_get_name ();
size_t instance_num = threadlocal_get_instance_num ();
int tty;
@@ -69,7 +66,7 @@ log_stderr_verror (const char *fs, va_list args)
}
fprintf (stderr, "error: ");
- errno = err; /* must restore in case fs contains %m */
+ errno = orig_errno; /* must restore in case fs contains %m */
vfprintf (stderr, fs, args);
fprintf (stderr, "\n");
@@ -78,6 +75,4 @@ log_stderr_verror (const char *fs, va_list args)
#ifdef HAVE_FUNLOCKFILE
funlockfile (stderr);
#endif
-
- errno = err; /* must be last line of function */
}
diff --git a/server/log-syslog.c b/server/log-syslog.c
index 76c5035b..29a7a825 100644
--- a/server/log-syslog.c
+++ b/server/log-syslog.c
@@ -45,11 +45,9 @@
/* Tempted to use LOG_FTP instead of LOG_DAEMON! */
static const int PRIORITY = LOG_DAEMON|LOG_ERR;
-/* Note: preserves the previous value of errno. */
void
-log_syslog_verror (const char *fs, va_list args)
+log_syslog_verror (int orig_errno, const char *fs, va_list args)
{
- int err = errno;
const char *name = threadlocal_get_name ();
size_t instance_num = threadlocal_get_instance_num ();
CLEANUP_FREE char *msg = NULL;
@@ -59,9 +57,9 @@ log_syslog_verror (const char *fs, va_list args)
fp = open_memstream (&msg, &len);
if (fp == NULL) {
/* Fallback to logging using fs, args directly. */
- errno = err; /* Must restore in case fs contains %m */
+ errno = orig_errno; /* must restore in case fs contains %m */
vsyslog (PRIORITY, fs, args);
- goto out;
+ return;
}
if (name) {
@@ -71,12 +69,9 @@ log_syslog_verror (const char *fs, va_list args)
fprintf (fp, ": ");
}
- errno = err; /* Must restore in case fs contains %m */
+ errno = orig_errno; /* must restore in case fs contains %m */
vfprintf (fp, fs, args);
close_memstream (fp);
syslog (PRIORITY, "%s", msg);
-
- out:
- errno = err;
}
diff --git a/server/log.c b/server/log.c
index 464e4f9a..9c1f667a 100644
--- a/server/log.c
+++ b/server/log.c
@@ -46,23 +46,27 @@
void
log_verror (const char *fs, va_list args)
{
+ int orig_errno = errno;
+
switch (log_to) {
case LOG_TO_DEFAULT:
if (forked_into_background)
- log_syslog_verror (fs, args);
+ log_syslog_verror (orig_errno, fs, args);
else
- log_stderr_verror (fs, args);
+ log_stderr_verror (orig_errno, fs, args);
break;
case LOG_TO_SYSLOG:
- log_syslog_verror (fs, args);
+ log_syslog_verror (orig_errno, fs, args);
break;
case LOG_TO_STDERR:
- log_stderr_verror (fs, args);
+ log_stderr_verror (orig_errno, fs, args);
break;
case LOG_TO_NULL:
/* nothing */
break;
}
+
+ errno = orig_errno; /* Restore errno before leaving the function. */
}
/* Note: preserves the previous value of errno. */
--
2.43.0

70
SOURCES/0002-tests-Use-exit-with-parent-in-the-test-framework.patch
Unescape View File

@ -0,0 +1,70 @@
From 93a2c7321dc5ccd36368d50887cf239cfb883a72 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 21 Jul 2023 19:01:35 +0100
Subject: [PATCH] tests: Use --exit-with-parent in the test framework
Make sure nbdkit always exits, in case some other part of the test is
killed.
(cherry picked from commit c6299889e80217cfadf488a67961be9eb6d24e38)
---
tests/Makefile.am | 2 ++
tests/test.c | 22 +++++++++++++---------
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/tests/Makefile.am b/tests/Makefile.am
index e1087689..f6c5ac9a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -594,6 +594,8 @@ EXTRA_DIST += \
check_LTLIBRARIES += libtest.la
libtest_la_SOURCES = test.c test.h
libtest_la_CFLAGS = $(WARNINGS_CFLAGS)
+libtest_la_CPPFLAGS = -I$(top_srcdir)/common/utils
+libtest_la_LIBADD = $(top_builddir)/common/utils/libutils.la
# Basic connection test.
LIBNBD_TESTS += test-connect
diff --git a/tests/test.c b/tests/test.c
index 67f69fab..6be10f12 100644
--- a/tests/test.c
+++ b/tests/test.c
@@ -50,6 +50,8 @@
#include <sys/wait.h>
#endif
+#include "exit-with-parent.h"
+
#include "test.h"
#ifndef WIN32
@@ -161,15 +163,17 @@ test_start_nbdkit (const char *arg, ...)
const char *argv[MAX_ARGS+1];
va_list args;
- argv[0] = "nbdkit";
- argv[1] = "-U";
- argv[2] = kit->sockpath;
- argv[3] = "-P";
- argv[4] = kit->pidpath;
- argv[5] = "-f";
- argv[6] = "-v";
- argv[7] = arg;
- i = 8;
+ i = 0;
+ argv[i++] = "nbdkit";
+ argv[i++] = "-U";
+ argv[i++] = kit->sockpath;
+ argv[i++] = "-P";
+ argv[i++] = kit->pidpath;
+ argv[i++] = "-f";
+ argv[i++] = "-v";
+ if (can_exit_with_parent ())
+ argv[i++] = "--exit-with-parent";
+ argv[i++] = arg;
va_start (args, arg);
while ((p = va_arg (args, const char *)) != NULL) {
--
2.39.3

26
SOURCES/0003-protect-Fix-copy-and-paste-error-in-the-documentatio.patch
Unescape View File

@ -0,0 +1,26 @@
From e3a62ebad571d6512501941c49f7a8ab66ef5646 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 22 Jul 2023 09:50:44 +0100
Subject: [PATCH] protect: Fix copy and paste error in the documentation
(cherry picked from commit f04607c95c27c4342f41d083ad00ac02c579b391)
---
filters/protect/nbdkit-protect-filter.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/filters/protect/nbdkit-protect-filter.pod b/filters/protect/nbdkit-protect-filter.pod
index b5ffc7ca..aad72a68 100644
--- a/filters/protect/nbdkit-protect-filter.pod
+++ b/filters/protect/nbdkit-protect-filter.pod
@@ -102,7 +102,7 @@ Use C<nbdkit --dump-config> to find the location of C<$filterdir>.
=head1 VERSION
-C<nbdkit-offset-filter> first appeared in nbdkit 1.30.
+C<nbdkit-protect-filter> first appeared in nbdkit 1.30.
=head1 SEE ALSO
--
2.39.3

175
SOURCES/0003-server-Rename-threadlocal_-set-get-_error-to-._errno.patch
Unescape View File

@ -1,175 +0,0 @@
From 1d7f655726ad3483d0e8086741182aada7ae8595 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 24 Jul 2024 10:29:13 +0100
Subject: [PATCH] server: Rename threadlocal_{set,get}_error to .._errno
A simple mechanical change, to avoid confusion with
threadlocal_{set,get}_last_error introduced in the following commit.
---
server/internal.h | 4 ++--
server/plugins.c | 27 +++++++++++++--------------
server/protocol.c | 5 +++--
server/threadlocal.c | 4 ++--
4 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/server/internal.h b/server/internal.h
index 1a783e3f..8102ccde 100644
--- a/server/internal.h
+++ b/server/internal.h
@@ -569,8 +569,8 @@ extern void threadlocal_set_name (const char *name)
extern const char *threadlocal_get_name (void);
extern void threadlocal_set_instance_num (size_t instance_num);
extern size_t threadlocal_get_instance_num (void);
-extern void threadlocal_set_error (int err);
-extern int threadlocal_get_error (void);
+extern void threadlocal_set_errno (int err);
+extern int threadlocal_get_errno (void);
extern void *threadlocal_buffer (size_t size);
extern void threadlocal_set_conn (struct connection *conn);
extern struct connection *threadlocal_get_conn (void);
diff --git a/server/plugins.c b/server/plugins.c
index ca89ac7a..3c7df0d2 100644
--- a/server/plugins.c
+++ b/server/plugins.c
@@ -633,15 +633,14 @@ plugin_can_cache (struct context *c)
NBDKIT_DLL_PUBLIC void
nbdkit_set_error (int err)
{
- threadlocal_set_error (err);
+ threadlocal_set_errno (err);
}
-/* Grab the appropriate error value.
- */
+/* Grab the appropriate error value. */
static int
-get_error (struct backend_plugin *p)
+get_errno (struct backend_plugin *p)
{
- int ret = threadlocal_get_error ();
+ int ret = threadlocal_get_errno ();
if (!ret && p->plugin.errno_is_preserved != 0)
ret = errno;
@@ -664,7 +663,7 @@ plugin_pread (struct context *c,
else
r = p->plugin._pread_v1 (c->handle, buf, count, offset);
if (r == -1)
- *err = get_error (p);
+ *err = get_errno (p);
return r;
}
@@ -685,7 +684,7 @@ plugin_flush (struct context *c,
return -1;
}
if (r == -1)
- *err = get_error (p);
+ *err = get_errno (p);
return r;
}
@@ -715,7 +714,7 @@ plugin_pwrite (struct context *c,
if (r != -1 && need_flush)
r = plugin_flush (c, 0, err);
if (r == -1 && !*err)
- *err = get_error (p);
+ *err = get_errno (p);
return r;
}
@@ -744,7 +743,7 @@ plugin_trim (struct context *c,
if (r != -1 && need_flush)
r = plugin_flush (c, 0, err);
if (r == -1 && !*err)
- *err = get_error (p);
+ *err = get_errno (p);
return r;
}
@@ -782,7 +781,7 @@ plugin_zero (struct context *c,
else
emulate = true;
if (r == -1)
- *err = emulate ? EOPNOTSUPP : get_error (p);
+ *err = emulate ? EOPNOTSUPP : get_errno (p);
if (r == 0 || (*err != EOPNOTSUPP && *err != ENOTSUP))
goto done;
}
@@ -794,7 +793,7 @@ plugin_zero (struct context *c,
}
flags &= ~NBDKIT_FLAG_MAY_TRIM;
- threadlocal_set_error (0);
+ threadlocal_set_errno (0);
*err = 0;
while (count) {
@@ -814,7 +813,7 @@ plugin_zero (struct context *c,
if (r != -1 && need_flush)
r = plugin_flush (c, 0, err);
if (r == -1 && !*err)
- *err = get_error (p);
+ *err = get_errno (p);
return r;
}
@@ -839,7 +838,7 @@ plugin_extents (struct context *c,
r = -1;
}
if (r == -1)
- *err = get_error (p);
+ *err = get_errno (p);
return r;
}
@@ -859,7 +858,7 @@ plugin_cache (struct context *c,
r = p->plugin.cache (c->handle, count, offset, flags);
if (r == -1)
- *err = get_error (p);
+ *err = get_errno (p);
return r;
}
diff --git a/server/protocol.c b/server/protocol.c
index 9b63f789..677da05c 100644
--- a/server/protocol.c
+++ b/server/protocol.c
@@ -235,8 +235,9 @@ handle_request (uint16_t cmd, uint16_t flags, uint64_t offset, uint32_t count,
int err = 0;
/* Clear the error, so that we know if the plugin calls
- * nbdkit_set_error() or relied on errno. */
- threadlocal_set_error (0);
+ * nbdkit_set_error() or relied on errno.
+ */
+ threadlocal_set_errno (0);
switch (cmd) {
case NBD_CMD_READ:
diff --git a/server/threadlocal.c b/server/threadlocal.c
index 088fe55a..9bb656bc 100644
--- a/server/threadlocal.c
+++ b/server/threadlocal.c
@@ -154,7 +154,7 @@ threadlocal_get_instance_num (void)
}
void
-threadlocal_set_error (int err)
+threadlocal_set_errno (int err)
{
struct threadlocal *threadlocal = pthread_getspecific (threadlocal_key);
@@ -167,7 +167,7 @@ threadlocal_set_error (int err)
/* This preserves errno, for convenience.
*/
int
-threadlocal_get_error (void)
+threadlocal_get_errno (void)
{
int err = errno;
struct threadlocal *threadlocal = pthread_getspecific (threadlocal_key);
--
2.43.0

26
SOURCES/0004-docs-nbdkit-protocol.pod-Fix-manual-page-name.patch
Unescape View File

@ -0,0 +1,26 @@
From c6da52fd3d9a9e5e8ad4648bba686c37b5d3fae7 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 23 Jul 2023 20:15:23 +0100
Subject: [PATCH] docs/nbdkit-protocol.pod: Fix manual page name
Updates: commit dce32051318c3e8bd20825cc0db5df15f95d11fa
(cherry picked from commit 00aa248cec0a64ca293bbedffbf47426ee2b2340)
---
docs/nbdkit-protocol.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/nbdkit-protocol.pod b/docs/nbdkit-protocol.pod
index 7e89a607..dabdf23c 100644
--- a/docs/nbdkit-protocol.pod
+++ b/docs/nbdkit-protocol.pod
@@ -1,6 +1,6 @@
=head1 NAME
-nbdkit - which parts of the NBD protocol nbdkit supports
+nbdkit-protocol - which parts of the NBD protocol nbdkit supports
=head1 SYNOPSIS
--
2.39.3

93
SOURCES/0004-server-Introduce-threadlocal_-set-get-_last_error.patch
Unescape View File

@ -1,93 +0,0 @@
From fa5055ae2b9f96af941d697de39198c96ee2580a Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 24 Jul 2024 10:37:58 +0100
Subject: [PATCH] server: Introduce threadlocal_{set,get}_last_error
Plus a function to clear the last_error field.
---
server/internal.h | 3 +++
server/threadlocal.c | 40 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 43 insertions(+)
diff --git a/server/internal.h b/server/internal.h
index 8102ccde..c45384a6 100644
--- a/server/internal.h
+++ b/server/internal.h
@@ -571,6 +571,9 @@ extern void threadlocal_set_instance_num (size_t instance_num);
extern size_t threadlocal_get_instance_num (void);
extern void threadlocal_set_errno (int err);
extern int threadlocal_get_errno (void);
+extern void threadlocal_set_last_error (char *msg);
+extern void threadlocal_clear_last_error (void);
+extern const char *threadlocal_get_last_error (void);
extern void *threadlocal_buffer (size_t size);
extern void threadlocal_set_conn (struct connection *conn);
extern struct connection *threadlocal_get_conn (void);
diff --git a/server/threadlocal.c b/server/threadlocal.c
index 9bb656bc..74a3c4e5 100644
--- a/server/threadlocal.c
+++ b/server/threadlocal.c
@@ -56,6 +56,7 @@ struct threadlocal {
char *name; /* Can be NULL. */
size_t instance_num; /* Can be 0. */
int err;
+ char *last_error; /* Can be NULL. */
void *buffer; /* Can be NULL. */
size_t buffer_size;
struct connection *conn; /* Can be NULL. */
@@ -70,6 +71,7 @@ free_threadlocal (void *threadlocalv)
struct threadlocal *threadlocal = threadlocalv;
free (threadlocal->name);
+ free (threadlocal->last_error);
free (threadlocal->buffer);
free (threadlocal);
}
@@ -176,6 +178,44 @@ threadlocal_get_errno (void)
return threadlocal ? threadlocal->err : 0;
}
+/* Set the last_error field. The ownership of the 'msg' string is
+ * passed to the threadlocal and will be freed here.
+ */
+void
+threadlocal_set_last_error (char *msg)
+{
+ struct threadlocal *threadlocal = pthread_getspecific (threadlocal_key);
+
+ if (threadlocal) {
+ free (threadlocal->last_error);
+ threadlocal->last_error = msg;
+ }
+ else {
+ /* ... otherwise throw it away, it's informational. */
+ free (msg);
+ }
+}
+
+void
+threadlocal_clear_last_error (void)
+{
+ threadlocal_set_last_error (NULL);
+}
+
+/* Get the last_error field. If successful, this returns a non-NULL
+ * string. This is valid until something calls nbdkit_error() in the
+ * same thread, so it should be used quickly. Returning NULL is not
+ * necessarily an error. The last_error is informational and may not
+ * be available.
+ */
+const char *
+threadlocal_get_last_error (void)
+{
+ struct threadlocal *threadlocal = pthread_getspecific (threadlocal_key);
+
+ return threadlocal ? threadlocal->last_error : NULL;
+}
+
/* Return the single pread/pwrite buffer for this thread. The buffer
* size is increased to ‘size’ bytes if required.
*
--
2.43.0

110
SOURCES/0005-retry-request-Print-operation-we-are-retrying-in-deb.patch
Unescape View File

@ -0,0 +1,110 @@
From e206e3a645ee6d350df210d2ef95614d3f5d97e7 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 16:43:07 +0100
Subject: [PATCH] retry-request: Print operation we are retrying in debug
messages
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit dbc91fd2eac5eac77293e3deded92e11c0b8d6e3)
---
filters/retry-request/retry-request.c | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/filters/retry-request/retry-request.c b/filters/retry-request/retry-request.c
index ed566080..e5b8344c 100644
--- a/filters/retry-request/retry-request.c
+++ b/filters/retry-request/retry-request.c
@@ -100,15 +100,15 @@ retry_request_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
* The code between RETRY_START...RETRY_END must set r to 0 or -1 on
* success or failure. *err may also be implicitly assigned.
*/
-#define RETRY_START \
+#define RETRY_START(what) \
{ \
unsigned i; \
\
r = -1; \
for (i = 0; r == -1 && i <= retries; ++i) { \
if (i > 0) { \
- nbdkit_debug ("retry %u: waiting %u seconds before retrying", \
- i, delay); \
+ nbdkit_debug ("retry %u: waiting %u seconds before retrying %s",\
+ i, delay, what); \
if (nbdkit_nanosleep (delay, 0) == -1) { \
if (*err == 0) \
*err = errno; \
@@ -130,7 +130,7 @@ retry_request_open (nbdkit_next_open *next, nbdkit_context *nxdata,
if (retry_open_call) {
int *err = &errno; /* used by the RETRY_* macros */
- RETRY_START
+ RETRY_START("open")
r = next (nxdata, readonly, exportname);
RETRY_END;
}
@@ -148,7 +148,7 @@ retry_request_pread (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("pread")
r = next->pread (next, buf, count, offset, flags, err);
RETRY_END;
return r;
@@ -162,7 +162,7 @@ retry_request_pwrite (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("pwrite")
r = next->pwrite (next, buf, count, offset, flags, err);
RETRY_END;
return r;
@@ -176,7 +176,7 @@ retry_request_trim (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("trim")
r = next->trim (next, count, offset, flags, err);
RETRY_END;
return r;
@@ -189,7 +189,7 @@ retry_request_flush (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("flush")
r = next->flush (next, flags, err);
RETRY_END;
return r;
@@ -203,7 +203,7 @@ retry_request_zero (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("zero")
r = next->zero (next, count, offset, flags, err);
RETRY_END;
return r;
@@ -218,7 +218,7 @@ retry_request_extents (nbdkit_next *next,
CLEANUP_EXTENTS_FREE struct nbdkit_extents *extents2 = NULL;
int r;
- RETRY_START {
+ RETRY_START("extents") {
/* Each retry must begin with extents reset to the right beginning. */
nbdkit_extents_free (extents2);
extents2 = nbdkit_extents_new (offset, next->get_size (next));
@@ -254,7 +254,7 @@ retry_request_cache (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("cache")
r = next->cache (next, count, offset, flags, err);
RETRY_END;
return r;
--
2.39.3

93
SOURCES/0005-server-Take-a-thread-local-copy-of-the-last-call-to-.patch
Unescape View File

@ -1,93 +0,0 @@
From bfa6d4064cb74f429149d14ab4025b258fc95ec4 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 23 Jul 2024 15:28:06 +0100
Subject: [PATCH] server: Take a thread-local copy of the last call to
nbdkit_error
nbdkit_error has traditionally been a "fancy wrapper around fprintf"
(kind of, don't take that literally). It is encouraged that plugins
and filters do something like:
if (error) {
nbdkit_error ("oops, a bad thing happened");
return -1;
}
but we don't enforce this. Plugins might call nbdkit_error more than
once or not at all.
The point where we get to sending an error back over the wire to the
NBD client is long after the plugin returned above, and after
nbdkit_error was called.
Therefore in order to send errors back to the NBD client, we must keep
the last error message around.
This change simply modifies nbdkit_error to make a best-effort attempt
to save the last error message in thread-local storage.
We also clear the last error when a new request starts, to ensure that
we don't leak errors across different callbacks or connections.
---
server/log.c | 21 +++++++++++++++++++++
server/protocol.c | 5 +++++
2 files changed, 26 insertions(+)
diff --git a/server/log.c b/server/log.c
index 9c1f667a..acf14d57 100644
--- a/server/log.c
+++ b/server/log.c
@@ -40,6 +40,25 @@
#include "internal.h"
+/* Copy the error message to threadlocal. This is sent to callers
+ * which are using structured replies, but is for extra information
+ * only so don't fail if we are unable to copy it.
+ */
+static void
+copy_error_to_threadlocal (int orig_errno, const char *fs, va_list args)
+{
+ va_list args_copy;
+ char *msg;
+ int r;
+
+ va_copy (args_copy, args);
+ errno = orig_errno; /* must restore in case fs contains %m */
+ r = vasprintf (&msg, fs, args_copy);
+ va_end (args_copy);
+ if (r != -1 && msg)
+ threadlocal_set_last_error (msg); /* ownership passed to threadlocal */
+}
+
/* Call the right log_*_verror function depending on log_sink.
* Note: preserves the previous value of errno.
*/
@@ -48,6 +67,8 @@ log_verror (const char *fs, va_list args)
{
int orig_errno = errno;
+ copy_error_to_threadlocal (orig_errno, fs, args);
+
switch (log_to) {
case LOG_TO_DEFAULT:
if (forked_into_background)
diff --git a/server/protocol.c b/server/protocol.c
index 677da05c..d428bfc8 100644
--- a/server/protocol.c
+++ b/server/protocol.c
@@ -239,6 +239,11 @@ handle_request (uint16_t cmd, uint16_t flags, uint64_t offset, uint32_t count,
*/
threadlocal_set_errno (0);
+ /* Also clear the last error in this thread so we will only save
+ * nbdkit_error() from this request.
+ */
+ threadlocal_clear_last_error ();
+
switch (cmd) {
case NBD_CMD_READ:
if (backend_pread (c, buf, count, offset, 0, &err) == -1)
--
2.43.0

175
SOURCES/0006-server-Send-the-last-error-to-the-NBD-client.patch
Unescape View File

@ -1,175 +0,0 @@
From 46484ca8e6a35c45fe96b6c972ceba8984d401e8 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 23 Jul 2024 15:45:04 +0100
Subject: [PATCH] server: Send the last error to the NBD client
This sends the last error saved in the connection handle back to the
NBD client. This is informational and best effort.
qemu reports the error already, for example:
$ nbdkit --log=null \
eval open=' echo EPERM Go Away >&2; exit 1 ' get_size=' echo 100 ' \
--run 'qemu-img info "$uri"'
qemu-img: Could not open 'nbd+unix://?socket=/tmp/nbdkitIDl6iy/socket': Requested export not available
server reported: /tmp/nbdkitRDAfXH/open: Go Away
This goes back to at least qemu 2.12.0 (RHEL 7) and possibly earlier,
so we can just assume that qemu does this for the test.
libnbd requires a patch to display this information.
---
server/protocol-handshake-newstyle.c | 43 ++++++++++++++++------
tests/Makefile.am | 2 +
tests/test-last-error.sh | 55 ++++++++++++++++++++++++++++
3 files changed, 88 insertions(+), 12 deletions(-)
create mode 100755 tests/test-last-error.sh
diff --git a/server/protocol-handshake-newstyle.c b/server/protocol-handshake-newstyle.c
index 6b3bc76f..c18d32e5 100644
--- a/server/protocol-handshake-newstyle.c
+++ b/server/protocol-handshake-newstyle.c
@@ -57,28 +57,47 @@ send_newstyle_option_reply (uint32_t option, uint32_t reply)
{
GET_CONN;
struct nbd_fixed_new_option_reply fixed_new_option_reply;
+ const char *last_error = NULL;
+ uint32_t replylen = 0;
+
+ if (NBD_REP_IS_ERR (reply)) {
+ last_error = threadlocal_get_last_error ();
+ /* Note that calling nbdkit_error will invalidate last_error, so
+ * be careful below.
+ */
+ if (last_error) {
+ size_t len = strlen (last_error);
+ if (len <= NBD_MAX_STRING)
+ replylen = len;
+ }
+ }
fixed_new_option_reply.magic = htobe64 (NBD_REP_MAGIC);
fixed_new_option_reply.option = htobe32 (option);
fixed_new_option_reply.reply = htobe32 (reply);
- fixed_new_option_reply.replylen = htobe32 (0);
+ fixed_new_option_reply.replylen = htobe32 (replylen);
debug ("replying to %s with %s", name_of_nbd_opt (option),
name_of_nbd_rep (reply));
if (conn->send (&fixed_new_option_reply,
- sizeof fixed_new_option_reply, 0) == -1) {
- /* The protocol document says that the client is allowed to simply
- * drop the connection after sending NBD_OPT_ABORT, or may read
- * the reply.
- */
- if (option == NBD_OPT_ABORT)
- debug ("write: %s: %m", name_of_nbd_opt (option));
- else
- nbdkit_error ("write: %s: %m", name_of_nbd_opt (option));
- return -1;
- }
+ sizeof fixed_new_option_reply,
+ replylen > 0 ? SEND_MORE : 0) == -1)
+ goto err;
+ if (replylen > 0 && conn->send (last_error, replylen, 0) == -1)
+ goto err;
return 0;
+
+err:
+ /* The protocol document says that the client is allowed to simply
+ * drop the connection after sending NBD_OPT_ABORT, or may read
+ * the reply.
+ */
+ if (option == NBD_OPT_ABORT)
+ debug ("write: %s: %m", name_of_nbd_opt (option));
+ else
+ nbdkit_error ("write: %s: %m", name_of_nbd_opt (option));
+ return -1;
}
/* Reply to NBD_OPT_LIST with the plugin's list of export names.
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 8c7d6b8c..89c5fa9d 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -292,6 +292,7 @@ TESTS += \
test-read-password-interactive.sh \
test-nbd-client.sh \
test-nbd-client-tls.sh \
+ test-last-error.sh \
$(NULL)
if !IS_WINDOWS
TESTS += \
@@ -324,6 +325,7 @@ EXTRA_DIST += \
test-help-plugin.sh \
test-ipv4-lo.sh \
test-ipv6-lo.sh \
+ test-last-error.sh \
test-long-name.sh \
test-nbd-client-tls.sh \
test-nbd-client.sh \
diff --git a/tests/test-last-error.sh b/tests/test-last-error.sh
new file mode 100755
index 00000000..fc720606
--- /dev/null
+++ b/tests/test-last-error.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright Red Hat
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+# Test informational error messages sent to the NBD client.
+# qemu-img supports this since at least 2.12.0.
+
+requires_run
+requires_plugin eval
+requires qemu-img --version
+
+out=last-error.out
+rm -f $out
+cleanup_fn rm -f $out
+
+export out
+
+nbdkit eval \
+ open=' echo EPERM Go Away >&2; exit 1 ' get_size=' echo 0 ' \
+ --run ' qemu-img info "$uri" > $out 2>&1 ||: '
+cat $out
+
+grep "Go Away" $out
--
2.43.0

53
SOURCES/0006-tests-test-ocaml-errorcodes.c-Don-t-use-assert-in-te.patch
Unescape View File

@ -0,0 +1,53 @@
From fdd27622047cd8e25af33b66a57899b75e99db82 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 1 Aug 2023 11:42:58 +0100
Subject: [PATCH] tests/test-ocaml-errorcodes.c: Don't use assert in test
exit with EXIT_FAILURE instead, as the program might be compiled with
-DNDEBUG.
(cherry picked from commit 678410e23841376d8b742dce3d10c632499367a2)
---
tests/test-ocaml-errorcodes.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/tests/test-ocaml-errorcodes.c b/tests/test-ocaml-errorcodes.c
index e59f074b..8ab7e0ae 100644
--- a/tests/test-ocaml-errorcodes.c
+++ b/tests/test-ocaml-errorcodes.c
@@ -34,8 +34,9 @@
#include <stdio.h>
#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
#include <errno.h>
-#include <assert.h>
#include <libnbd.h>
@@ -84,10 +85,19 @@ main (int argc, char *argv[])
exit (EXIT_FAILURE);
}
- assert (nbd_pread (nbd, buf, 512, 0, 0) == 0);
+ if (nbd_pread (nbd, buf, 512, 0, 0) == -1) {
+ fprintf (stderr, "%s: FAIL: did not expect reading sector 0 to fail\n",
+ argv[0]);
+ exit (EXIT_FAILURE);
+ }
for (i = 0; tests[i].offset != 0; ++i) {
- assert (nbd_pread (nbd, buf, 512, tests[i].offset, 0) == -1);
+ if (nbd_pread (nbd, buf, 512, tests[i].offset, 0) != -1) {
+ fprintf (stderr,
+ "%s: FAIL: reading sector %" PRIu64 "should have failed\n",
+ argv[0], tests[i].offset / 512);
+ exit (EXIT_FAILURE);
+ }
actual_errno = nbd_get_errno ();
if (actual_errno != tests[i].expected_errno) {
fprintf (stderr, "%s: FAIL: actual errno = %d expected errno = %d\n",
--
2.39.3

30
SOURCES/0007-tests-test-ocaml-errorcodes.c-Enable-verbose-message.patch
Unescape View File

@ -0,0 +1,30 @@
From eca5131cbbc7e1785b266cc92624a2ce9582a7cc Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 1 Aug 2023 12:42:05 +0100
Subject: [PATCH] tests/test-ocaml-errorcodes.c: Enable verbose messages
This test crashes when the OCaml plugin is unloaded (specifically when
we call caml_acquire_runtime_system). Enable verbose messages in a
failed attempt to track down what is happening.
(cherry picked from commit 99567408fdd8ea55b0bb5286b45ce135d91c38b5)
---
tests/test-ocaml-errorcodes.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/test-ocaml-errorcodes.c b/tests/test-ocaml-errorcodes.c
index 8ab7e0ae..f84da3d8 100644
--- a/tests/test-ocaml-errorcodes.c
+++ b/tests/test-ocaml-errorcodes.c
@@ -78,7 +78,7 @@ main (int argc, char *argv[])
if (nbd_connect_command (nbd,
(char *[]) {
- "nbdkit", "-s", "--exit-with-parent",
+ "nbdkit", "-s", "--exit-with-parent", "-v",
"./test-ocaml-errorcodes-plugin.so",
NULL }) == -1) {
fprintf (stderr, "%s\n", nbd_get_error ());
--
2.39.3

80
SOURCES/0008-curl-Use-the-parallel-thread-model.patch
Unescape View File

@ -0,0 +1,80 @@
From bfa0b1238f6318d06930ec4d389a27932fc90a16 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 4 Feb 2023 12:34:57 +0000
Subject: [PATCH] curl: Use the parallel thread model
After previous changes, it is now safe to use the parallel thread
model in this plugin. The locking in pool.c protects a single curl
handle from being used from multiple threads.
An advantage of this is we can now combine the curl plugin with
filters such as readahead and scan.
This pessimizes some workloads and improves others. See my earlier
comments here:
https://listman.redhat.com/archives/libguestfs/2023-February/030610.html
https://listman.redhat.com/archives/libguestfs/2023-February/030618.html
https://listman.redhat.com/archives/libguestfs/2023-February/030619.html
https://listman.redhat.com/archives/libguestfs/2023-February/030620.html
Tests below use this basic command:
$ time nbdkit -r -U - curl https://cloud.debian.org/images/cloud/bookworm/daily/latest/debian-12-generic-amd64-daily.qcow2 \
--run '$COPY $uri /var/tmp/out'
where $COPY is either nbdcopy or qemu-img convert.
Before this change:
nbdkit + nbdcopy 0m55.397s
nbdkit + qemu-img convert [over 20 minutes]
After this change:
nbdkit + nbdcopy 1m1.235s
nbdkit + qemu-img convert 6m3.262s
nbdkit --filter=readahead --filter=cache + qemu-img convert
8m16.488s
nbdkit connections=8 + qemu-img convert
3m48.502s
Previously [see first link above] we noted that file: URLs are
impacted, and indeed they are, with this command:
$ time nbdkit -r -U - curl file:/var/tmp/fedora-36.img \
--run 'nbdcopy --no-extents -p "$uri" null:'
nearly doubling in run-time (0.78 -> 1.34). However I think this may
be a peculiarity of curl's handling of file. (Use nbdkit-file-plugin
instead).
Note the fundamental issue here is still lack of multiconn support in
qemu's NBD client.
(cherry picked from commit f2163754860d041c4cb12dace90591c280eccae8)
---
plugins/curl/curl.c | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 58ffb662..425f1d90 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -537,12 +537,7 @@ curl_close (void *handle)
free (h);
}
-/* This plugin could support the parallel thread model. It currently
- * uses serialize_requests because parallel has the unfortunate effect
- * of pessimising common workloads. See:
- * https://listman.redhat.com/archives/libguestfs/2023-February/030618.html
- */
-#define THREAD_MODEL NBDKIT_THREAD_MODEL_SERIALIZE_REQUESTS
+#define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL
/* Calls get_handle() ... put_handle() to get a handle for the length
* of the current scope.
--
2.39.3

115
SOURCES/0009-curl-Add-ipresolve-option.patch
Unescape View File

@ -0,0 +1,115 @@
From a5f1e659d52872a499a5744870cb3e6ff382642f Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 25 Jul 2023 16:57:41 +0100
Subject: [PATCH] curl: Add ipresolve option
Allows you to force IPv4 or IPv6.
(cherry picked from commit d1b27c97f7fac00ee452a1d1b05805ee7145a49c)
---
plugins/curl/curl.c | 15 +++++++++++++++
plugins/curl/curldefs.h | 1 +
plugins/curl/nbdkit-curl-plugin.pod | 14 ++++++++++++++
plugins/curl/pool.c | 2 ++
4 files changed, 32 insertions(+)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 425f1d90..91fa65fb 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -68,6 +68,7 @@ struct curl_slist *headers = NULL;
const char *header_script = NULL;
unsigned header_script_renew = 0;
long http_version = CURL_HTTP_VERSION_NONE;
+long ipresolve = CURL_IPRESOLVE_WHATEVER;
char *password = NULL;
#ifndef HAVE_CURLOPT_PROTOCOLS_STR
long protocols = CURLPROTO_ALL;
@@ -314,6 +315,19 @@ curl_config (const char *key, const char *value)
}
}
+ else if (strcmp (key, "ipresolve") == 0) {
+ if (strcmp (value, "any") == 0 || strcmp (value, "whatever") == 0)
+ ipresolve = CURL_IPRESOLVE_WHATEVER;
+ else if (strcmp (value, "v4") == 0 || strcmp (value, "4") == 0)
+ ipresolve = CURL_IPRESOLVE_V4;
+ else if (strcmp (value, "v6") == 0 || strcmp (value, "6") == 0)
+ ipresolve = CURL_IPRESOLVE_V6;
+ else {
+ nbdkit_error ("unknown ipresolve: %s", value);
+ return -1;
+ }
+ }
+
else if (strcmp (key, "password") == 0) {
free (password);
if (nbdkit_read_password (value, &password) == -1)
@@ -495,6 +509,7 @@ curl_config_complete (void)
"header-script=<SCRIPT> Script to set HTTP/HTTPS headers.\n" \
"header-script-renew=<SECS> Time to renew HTTP/HTTPS headers.\n" \
"http-version=none|... Force a particular HTTP protocol.\n" \
+ "ipresolve=any|v4|v6 Force IPv4 or IPv6.\n" \
"password=<PASSWORD> The password for the user account.\n" \
"protocols=PROTO,PROTO,.. Limit protocols allowed.\n" \
"proxy=<PROXY> Set proxy URL.\n" \
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 2a74a369..815be2e1 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -61,6 +61,7 @@ extern struct curl_slist *headers;
extern const char *header_script;
extern unsigned header_script_renew;
extern long http_version;
+extern long ipresolve;
extern char *password;
#ifndef HAVE_CURLOPT_PROTOCOLS_STR
extern long protocols;
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index 070f9a0f..e12ca197 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -208,6 +208,19 @@ meaning curl will negotiate the best protocol with the server. The
other settings are mainly for testing. See L<CURLOPT_HTTP_VERSION(3)>
for details.
+=item B<ipresolve=any>
+
+=item B<ipresolve=v4>
+
+=item B<ipresolve=v6>
+
+(nbdkit E<ge> 1.36)
+
+Force curl to use only IPv4 (C<ipresolve=v4>), only IPv6
+(C<ipresolve=v6>) or any IP version supported by your system
+(C<ipresolve=any>). The default is C<any>. See
+L<CURLOPT_IPRESOLVE(3)>.
+
=item B<password=>PASSWORD
Set the password to use when connecting to the remote server.
@@ -559,6 +572,7 @@ L<CURLOPT_COOKIEFILE(3)>,
L<CURLOPT_COOKIEJAR(3)>,
L<CURLOPT_FOLLOWLOCATION(3)>,
L<CURLOPT_HTTPHEADER(3)>,
+L<CURLOPT_IPRESOLVE(3)>,
L<CURLOPT_PROXY(3)>,
L<CURLOPT_SSL_CIPHER_LIST(3)>,
L<CURLOPT_SSLVERSION(3)>,
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 731dd367..f0c3cb4f 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -257,6 +257,8 @@ allocate_handle (void)
curl_easy_setopt (ch->c, CURLOPT_HTTPHEADER, headers);
if (http_version != CURL_HTTP_VERSION_NONE)
curl_easy_setopt (ch->c, CURLOPT_HTTP_VERSION, (long) http_version);
+ if (ipresolve != CURL_IPRESOLVE_WHATEVER)
+ curl_easy_setopt (ch->c, CURLOPT_IPRESOLVE, (long) ipresolve);
if (password)
curl_easy_setopt (ch->c, CURLOPT_PASSWORD, password);
--
2.39.3

113
SOURCES/0010-curl-Add-resolve-option.patch
Unescape View File

@ -0,0 +1,113 @@
From 55e55ea986ef5fed595bb5a4203e8734d79f1474 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 25 Jul 2023 17:36:31 +0100
Subject: [PATCH] curl: Add resolve option
This allows you to force a particular IP address for the URL host
name.
(cherry picked from commit 4f0989cbf0e9eeb959879b1c82b52940f6c5c3cc)
---
plugins/curl/curl.c | 12 ++++++++++++
plugins/curl/curldefs.h | 1 +
plugins/curl/nbdkit-curl-plugin.pod | 7 +++++++
plugins/curl/pool.c | 2 ++
4 files changed, 22 insertions(+)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 91fa65fb..381433fd 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -78,6 +78,7 @@ const char *protocols = NULL;
const char *proxy = NULL;
char *proxy_password = NULL;
const char *proxy_user = NULL;
+struct curl_slist *resolves = NULL;
bool sslverify = true;
const char *ssl_cipher_list = NULL;
long ssl_version = CURL_SSLVERSION_DEFAULT;
@@ -112,6 +113,8 @@ curl_unload (void)
curl_slist_free_all (headers);
free (password);
free (proxy_password);
+ if (resolves)
+ curl_slist_free_all (resolves);
scripts_unload ();
free_all_handles ();
curl_global_cleanup ();
@@ -356,6 +359,14 @@ curl_config (const char *key, const char *value)
else if (strcmp (key, "proxy-user") == 0)
proxy_user = value;
+ else if (strcmp (key, "resolve") == 0) {
+ resolves = curl_slist_append (headers, value);
+ if (resolves == NULL) {
+ nbdkit_error ("curl_slist_append: %m");
+ return -1;
+ }
+ }
+
else if (strcmp (key, "sslverify") == 0) {
r = nbdkit_parse_bool (value);
if (r == -1)
@@ -515,6 +526,7 @@ curl_config_complete (void)
"proxy=<PROXY> Set proxy URL.\n" \
"proxy-password=<PASSWORD> The proxy password.\n" \
"proxy-user=<USER> The proxy user.\n" \
+ "resolve=<HOST>:<PORT>:<ADDR> Custom host to IP address resolution.\n" \
"sslverify=false Do not verify SSL certificate of remote host.\n" \
"ssl-cipher-list=C1:C2:.. Specify TLS/SSL cipher suites to be used.\n" \
"ssl-version=<VERSION> Specify preferred TLS/SSL version.\n" \
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 815be2e1..613cfed7 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -74,6 +74,7 @@ extern const char *proxy_user;
extern bool sslverify;
extern const char *ssl_cipher_list;
extern long ssl_version;
+extern struct curl_slist *resolves;
extern const char *tls13_ciphers;
extern bool tcp_keepalive;
extern bool tcp_nodelay;
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index e12ca197..a7315047 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -289,6 +289,12 @@ Set the proxy. See L<CURLOPT_PROXY(3)>.
Set the proxy username and password.
+=item B<resolve=>HOSTB<:>PORTB<:>ADDRESS
+
+Provide custom host name to IP address resolution. You can supply
+this option as many times as needed. See L<CURLOPT_RESOLVE(3)> for
+the full details of this option.
+
=item B<sslverify=false>
Don't verify the SSL certificate of the remote host.
@@ -574,6 +580,7 @@ L<CURLOPT_FOLLOWLOCATION(3)>,
L<CURLOPT_HTTPHEADER(3)>,
L<CURLOPT_IPRESOLVE(3)>,
L<CURLOPT_PROXY(3)>,
+L<CURLOPT_RESOLVE(3)>,
L<CURLOPT_SSL_CIPHER_LIST(3)>,
L<CURLOPT_SSLVERSION(3)>,
L<CURLOPT_TCP_KEEPALIVE(3)>,
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index f0c3cb4f..a6e2f9f5 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -283,6 +283,8 @@ allocate_handle (void)
curl_easy_setopt (ch->c, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt (ch->c, CURLOPT_SSL_VERIFYHOST, 0L);
}
+ if (resolves)
+ curl_easy_setopt (ch->c, CURLOPT_RESOLVE, resolves);
if (ssl_version != CURL_SSLVERSION_DEFAULT)
curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, (long) ssl_version);
if (ssl_cipher_list)
--
2.39.3

82
SOURCES/0011-curl-pool-Add-abstract-load_pool-and-unload_pool-fun.patch
Unescape View File

@ -0,0 +1,82 @@
From f14fbc9498f52661fd8b9c895144eb40bf8bcd64 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 25 Jul 2023 18:57:00 +0100
Subject: [PATCH] curl: pool: Add abstract load_pool and unload_pool functions
Simple code refactoring.
(cherry picked from commit 93761b2abf4c5923a410a4f176ee66c690520a2c)
---
plugins/curl/curl.c | 4 +++-
plugins/curl/curldefs.h | 3 ++-
plugins/curl/pool.c | 10 ++++++++--
3 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 381433fd..3557fd53 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -103,6 +103,8 @@ curl_load (void)
nbdkit_error ("libcurl initialization failed: %d", (int) r);
exit (EXIT_FAILURE);
}
+
+ load_pool ();
}
static void
@@ -116,7 +118,7 @@ curl_unload (void)
if (resolves)
curl_slist_free_all (resolves);
scripts_unload ();
- free_all_handles ();
+ unload_pool ();
curl_global_cleanup ();
}
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 613cfed7..4506f3e1 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -122,9 +122,10 @@ struct curl_handle {
};
/* pool.c */
+extern void load_pool (void);
+extern void unload_pool (void);
extern struct curl_handle *get_handle (void);
extern void put_handle (struct curl_handle *ch);
-extern void free_all_handles (void);
/* scripts.c */
extern int do_scripts (struct curl_handle *ch);
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index a6e2f9f5..f91cdf57 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -89,14 +89,20 @@ static curl_handle_list curl_handles = empty_vector;
static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
static size_t in_use = 0, waiting = 0;
+/* Initialize pool structures. */
+void
+load_pool (void)
+{
+}
+
/* Close and free all handles in the pool. */
void
-free_all_handles (void)
+unload_pool (void)
{
size_t i;
if (curl_debug_pool)
- nbdkit_debug ("free_all_handles: number of curl handles allocated: %zu",
+ nbdkit_debug ("unload_pool: number of curl handles allocated: %zu",
curl_handles.len);
for (i = 0; i < curl_handles.len; ++i)
--
2.39.3

278
SOURCES/0012-curl-Add-D-curl.times-1-to-collect-time-statistics.patch
Unescape View File

@ -0,0 +1,278 @@
From d8bbc2dd00eddd2c36a6a718f7e21e6e7d4f26e5 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 25 Jul 2023 21:45:31 +0100
Subject: [PATCH] curl: Add -D curl.times=1 to collect time statistics
(cherry picked from commit 68dddbeb584fb9385915846d259563f74338ffe8)
---
plugins/curl/Makefile.am | 1 +
plugins/curl/curl.c | 3 +
plugins/curl/curldefs.h | 13 +++
plugins/curl/nbdkit-curl-plugin.pod | 21 +++++
plugins/curl/pool.c | 2 +
plugins/curl/times.c | 130 ++++++++++++++++++++++++++++
6 files changed, 170 insertions(+)
create mode 100644 plugins/curl/times.c
diff --git a/plugins/curl/Makefile.am b/plugins/curl/Makefile.am
index 82c09582..f5f1cc1a 100644
--- a/plugins/curl/Makefile.am
+++ b/plugins/curl/Makefile.am
@@ -42,6 +42,7 @@ nbdkit_curl_plugin_la_SOURCES = \
curl.c \
pool.c \
scripts.c \
+ times.c \
$(top_srcdir)/include/nbdkit-plugin.h \
$(NULL)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 3557fd53..0d1fcfee 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -119,6 +119,7 @@ curl_unload (void)
curl_slist_free_all (resolves);
scripts_unload ();
unload_pool ();
+ display_times ();
curl_global_cleanup ();
}
@@ -639,6 +640,7 @@ curl_pread (void *handle, void *buf, uint32_t count, uint64_t offset)
display_curl_error (ch, r, "pread: curl_easy_perform");
return -1;
}
+ update_times (ch->c);
/* Could use curl_easy_getinfo here to obtain further information
* about the connection.
@@ -683,6 +685,7 @@ curl_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset)
display_curl_error (ch, r, "pwrite: curl_easy_perform");
return -1;
}
+ update_times (ch->c);
/* Could use curl_easy_getinfo here to obtain further information
* about the connection.
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 4506f3e1..dd9791aa 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -44,6 +44,15 @@
#if CURL_AT_LEAST_VERSION (7, 55, 0)
#define HAVE_CURLINFO_CONTENT_LENGTH_DOWNLOAD_T
#endif
+#if CURL_AT_LEAST_VERSION (7, 61, 0)
+#define HAVE_CURLINFO_NAMELOOKUP_TIME_T
+#define HAVE_CURLINFO_CONNECT_TIME_T
+#define HAVE_CURLINFO_APPCONNECT_TIME_T
+#define HAVE_CURLINFO_PRETRANSFER_TIME_T
+#define HAVE_CURLINFO_STARTTRANSFER_TIME_T
+#define HAVE_CURLINFO_TOTAL_TIME_T
+#define HAVE_CURLINFO_REDIRECT_TIME_T
+#endif
#endif
extern const char *url;
@@ -131,6 +140,10 @@ extern void put_handle (struct curl_handle *ch);
extern int do_scripts (struct curl_handle *ch);
extern void scripts_unload (void);
+/* times.c */
+extern void update_times (CURL *c); /* called after every curl_easy_perform */
+extern void display_times (void);
+
/* Translate CURLcode to nbdkit_error. */
#define display_curl_error(ch, r, fs, ...) \
do { \
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index a7315047..0fd688ed 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -543,6 +543,27 @@ This prints out the headers and cookies generated by the
C<header-script> and C<cookie-script> options, which can be useful
when debugging these scripts.
+=item B<-D curl.times=1>
+
+This prints out additional information about the total time taken to
+do name resolution, connect to the remote server, etc. The
+information is printed in the debug output before nbdkit exits. The
+output will look like:
+
+ nbdkit: debug: times (-D curl.times=1):
+ nbdkit: debug: name resolution : 0.128442 s
+ nbdkit: debug: connection : 4.945213 s
+ nbdkit: debug: SSL negotiation : 4.291362 s
+ nbdkit: debug: pretransfer : 0.104137 s
+ nbdkit: debug: first byte received : 56.115269 s
+ nbdkit: debug: data transfer : 222.633831 s
+ nbdkit: debug: redirection time : 0.000000 s
+
+The cumulative time taken to perform each step is shown (summed across
+all HTTP connections). The redirection time is the total time taken
+doing HTTP redirections. For further information see
+L<curl_easy_getinfo(3)/TIMES>.
+
=item B<-D curl.verbose=1>
This enables very verbose curl debugging. See L<CURLOPT_VERBOSE(3)>.
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index f91cdf57..10f9011d 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -475,6 +475,7 @@ get_content_length_accept_range (struct curl_handle *ch)
display_curl_error (ch, r,
"problem doing HEAD request to fetch size of URL [%s]",
url);
+ update_times (ch->c);
/* Get the HTTP status code, if available. */
r = curl_easy_getinfo (ch->c, CURLINFO_RESPONSE_CODE, &code);
@@ -569,6 +570,7 @@ try_fallback_GET_method (struct curl_handle *ch)
curl_easy_setopt (ch->c, CURLOPT_WRITEFUNCTION, error_cb);
curl_easy_setopt (ch->c, CURLOPT_WRITEDATA, ch);
r = curl_easy_perform (ch->c);
+ update_times (ch->c);
/* We expect CURLE_WRITE_ERROR here, but CURLE_OK is possible too
* (eg if the remote has zero length). Other errors might happen
diff --git a/plugins/curl/times.c b/plugins/curl/times.c
new file mode 100644
index 00000000..8cc4cf27
--- /dev/null
+++ b/plugins/curl/times.c
@@ -0,0 +1,130 @@
+/* nbdkit
+ * Copyright Red Hat
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <curl/curl.h>
+
+#include <nbdkit-plugin.h>
+
+#include "array-size.h"
+
+#include "curldefs.h"
+
+/* Use '-D curl.times=1' to set. */
+NBDKIT_DLL_PUBLIC int curl_debug_times = 0;
+
+/* The cumulative times. */
+static struct {
+ bool cumulative;
+ const char *name;
+ CURLINFO info;
+ curl_off_t t;
+} times[] = {
+#ifdef HAVE_CURLINFO_NAMELOOKUP_TIME_T
+ { true, "name resolution", CURLINFO_NAMELOOKUP_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_CONNECT_TIME_T
+ { true, "connection", CURLINFO_CONNECT_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_APPCONNECT_TIME_T
+ { true, "SSL negotiation", CURLINFO_APPCONNECT_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_PRETRANSFER_TIME_T
+ { true, "pretransfer", CURLINFO_PRETRANSFER_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_STARTTRANSFER_TIME_T
+ { true, "first byte received", CURLINFO_STARTTRANSFER_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_TOTAL_TIME_T
+ { true, "data transfer", CURLINFO_TOTAL_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_REDIRECT_TIME_T
+ { false, "redirection time", CURLINFO_REDIRECT_TIME_T },
+#endif
+};
+
+/* This is called after every curl_easy_perform. If -D curl.times=1
+ * then we update the time counters. Refer to curl_easy_getinfo(3)
+ * section "TIMES".
+ */
+void
+update_times (CURL *c)
+{
+ size_t i;
+ CURLcode r;
+ curl_off_t t;
+
+ if (!curl_debug_times) return;
+
+ for (i = 0; i < ARRAY_SIZE (times); ++i) {
+ r = curl_easy_getinfo (c, times[i].info, &t);
+ if (r != CURLE_OK) {
+ nbdkit_debug ("curl_easy_getinfo: error getting time '%s': %s",
+ times[i].name, curl_easy_strerror (r));
+ continue;
+ }
+ if (curl_debug_verbose)
+ nbdkit_debug ("time '%s': %" PRIi64, times[i].name, (int64_t) t);
+ times[i].t += t;
+ }
+}
+
+/* Called when the plugin is unloaded. */
+void
+display_times (void)
+{
+ size_t i;
+ int64_t prev_t = 0, t, v;
+
+ if (!curl_debug_times) return;
+
+ nbdkit_debug ("times (-D curl.times=1):");
+ for (i = 0; i < ARRAY_SIZE (times); ++i) {
+ t = times[i].t; /* in microseconds */
+ if (times[i].cumulative)
+ v = t - prev_t;
+ else
+ v = t;
+ prev_t = t;
+
+ nbdkit_debug ("%-30s: %3" PRIi64 ".%06" PRIi64 " s",
+ times[i].name,
+ v / 1000000, v % 1000000);
+ }
+}
--
2.39.3

34
SOURCES/0013-curl-Fix-call-to-update_times.patch
Unescape View File

@ -0,0 +1,34 @@
From 320af6bbc9d71e3e67cf1d6e59d4a5094324d41c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 27 Jul 2023 17:08:50 +0100
Subject: [PATCH] curl: Fix call to update_times
This was called in the wrong place so we didn't count the cost of
making the initial HEAD call on each handle.
Fixes: commit 68dddbeb584fb9385915846d259563f74338ffe8
(cherry picked from commit f2d7041f94af15b158c92a96b9d9fdf4d3b7cdef)
---
plugins/curl/pool.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 10f9011d..8db69a71 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -471,11 +471,11 @@ get_content_length_accept_range (struct curl_handle *ch)
curl_easy_setopt (ch->c, CURLOPT_HEADERFUNCTION, header_cb);
curl_easy_setopt (ch->c, CURLOPT_HEADERDATA, ch);
r = curl_easy_perform (ch->c);
+ update_times (ch->c);
if (r != CURLE_OK) {
display_curl_error (ch, r,
"problem doing HEAD request to fetch size of URL [%s]",
url);
- update_times (ch->c);
/* Get the HTTP status code, if available. */
r = curl_easy_getinfo (ch->c, CURLINFO_RESPONSE_CODE, &code);
--
2.39.3

2075
SOURCES/0014-curl-Move-configuration-code-to-a-separate-file.patch
View File

File diff suppressed because it is too large Load Diff

53
SOURCES/0015-curl-Make-times-seconds-field-slightly-wider.patch
Unescape View File

@ -0,0 +1,53 @@
From 110a96d728d54ff31d0d5ff1684427b45e91b832 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 11:26:49 +0100
Subject: [PATCH] curl: Make times seconds field slightly wider
Updates: commit 68dddbeb584fb9385915846d259563f74338ffe8
(cherry picked from commit 1622db8c9f6c6e42b679d1d81df019944178b373)
---
plugins/curl/nbdkit-curl-plugin.pod | 14 +++++++-------
plugins/curl/times.c | 2 +-
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index 0fd688ed..0774adad 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -551,13 +551,13 @@ information is printed in the debug output before nbdkit exits. The
output will look like:
nbdkit: debug: times (-D curl.times=1):
- nbdkit: debug: name resolution : 0.128442 s
- nbdkit: debug: connection : 4.945213 s
- nbdkit: debug: SSL negotiation : 4.291362 s
- nbdkit: debug: pretransfer : 0.104137 s
- nbdkit: debug: first byte received : 56.115269 s
- nbdkit: debug: data transfer : 222.633831 s
- nbdkit: debug: redirection time : 0.000000 s
+ nbdkit: debug: name resolution : 0.128442 s
+ nbdkit: debug: connection : 4.945213 s
+ nbdkit: debug: SSL negotiation : 4.291362 s
+ nbdkit: debug: pretransfer : 0.104137 s
+ nbdkit: debug: first byte received : 56.115269 s
+ nbdkit: debug: data transfer : 222.633831 s
+ nbdkit: debug: redirection time : 0.000000 s
The cumulative time taken to perform each step is shown (summed across
all HTTP connections). The redirection time is the total time taken
diff --git a/plugins/curl/times.c b/plugins/curl/times.c
index 8cc4cf27..e752a0a9 100644
--- a/plugins/curl/times.c
+++ b/plugins/curl/times.c
@@ -123,7 +123,7 @@ display_times (void)
v = t;
prev_t = t;
- nbdkit_debug ("%-30s: %3" PRIi64 ".%06" PRIi64 " s",
+ nbdkit_debug ("%-30s: %4" PRIi64 ".%06" PRIi64 " s",
times[i].name,
v / 1000000, v % 1000000);
}
--
2.39.3

51
SOURCES/0016-curl-Use-_Atomic-type-to-accumulate-curl-timings.patch
Unescape View File

@ -0,0 +1,51 @@
From 22590e0575e78c4fc754705b46d1458eeadefcf1 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 30 Jul 2023 16:45:38 +0100
Subject: [PATCH] curl: Use _Atomic type to accumulate curl timings
Because the global list of times is accessed in parallel by many
threads, they must be accessed atomically.
Fixes: commit 68dddbeb584fb9385915846d259563f74338ffe8
(cherry picked from commit 4c527063336ccf14d286ef7db5766369e1b23845)
---
plugins/curl/curldefs.h | 9 +++++++++
plugins/curl/times.c | 2 +-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 9169b256..1feba844 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -35,6 +35,15 @@
#include <stdbool.h>
+#ifdef HAVE_STDATOMIC_H
+#include <stdatomic.h>
+#else
+/* Some old platforms lack atomic types, but 32 bit ints are usually
+ * "atomic enough".
+ */
+#define _Atomic /**/
+#endif
+
#include "windows-compat.h"
/* Macro CURL_AT_LEAST_VERSION was added in 2015 (Curl 7.43) so if the
diff --git a/plugins/curl/times.c b/plugins/curl/times.c
index e752a0a9..23e2950b 100644
--- a/plugins/curl/times.c
+++ b/plugins/curl/times.c
@@ -54,7 +54,7 @@ static struct {
bool cumulative;
const char *name;
CURLINFO info;
- curl_off_t t;
+ _Atomic curl_off_t t;
} times[] = {
#ifdef HAVE_CURLINFO_NAMELOOKUP_TIME_T
{ true, "name resolution", CURLINFO_NAMELOOKUP_TIME_T },
--
2.39.3

120
SOURCES/0017-curl-Add-D-curl.verbose.ids-1-to-display-conn-and-xf.patch
Unescape View File

@ -0,0 +1,120 @@
From 07bdd644ae082c8e05afdcda4dcd12816ecc0efc Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 31 Jul 2023 13:24:39 +0100
Subject: [PATCH] curl: Add -D curl.verbose.ids=1 to display conn and xfer IDs
Enhance debugging output with connection (conn) and transfer (xfer)
IDs. Since there is some overhead to doing this, don't do it by
default. Also it requires libcurl >= 8.2.0.
(cherry picked from commit a3ebf8f1c2cd5d399730ad6887b6b8bed94dc0ce)
---
plugins/curl/config.c | 37 +++++++++++++++++++++++++----
plugins/curl/curldefs.h | 4 ++++
plugins/curl/nbdkit-curl-plugin.pod | 6 +++++
3 files changed, 43 insertions(+), 4 deletions(-)
diff --git a/plugins/curl/config.c b/plugins/curl/config.c
index 742d6080..d68c92ba 100644
--- a/plugins/curl/config.c
+++ b/plugins/curl/config.c
@@ -99,6 +99,9 @@ static size_t error_cb (char *ptr, size_t size, size_t nmemb, void *opaque);
/* Use '-D curl.verbose=1' to set. */
NBDKIT_DLL_PUBLIC int curl_debug_verbose = 0;
+/* Use '-D curl.verbose.ids=1' to set. */
+NBDKIT_DLL_PUBLIC int curl_debug_verbose_ids = 0;
+
void
unload_config (void)
{
@@ -707,6 +710,14 @@ debug_cb (CURL *handle, curl_infotype type,
{
size_t origsize = size;
CLEANUP_FREE char *str;
+ curl_off_t conn_id = -1, xfer_id = -1;
+
+#if defined(HAVE_CURLINFO_CONN_ID) && defined(HAVE_CURLINFO_XFER_ID)
+ if (curl_debug_verbose_ids) {
+ curl_easy_getinfo (handle, CURLINFO_CONN_ID, &conn_id);
+ curl_easy_getinfo (handle, CURLINFO_XFER_ID, &xfer_id);
+ }
+#endif
/* The data parameter passed is NOT \0-terminated, but also it may
* have \n or \r\n line endings. The only sane way to deal with
@@ -726,17 +737,35 @@ debug_cb (CURL *handle, curl_infotype type,
switch (type) {
case CURLINFO_TEXT:
- nbdkit_debug ("%s", str);
+ if (conn_id >= 0 && xfer_id >= 0)
+ nbdkit_debug ("conn %" PRIi64 " xfer %" PRIi64 ": %s",
+ conn_id, xfer_id, str);
+ else
+ nbdkit_debug ("%s",str);
break;
case CURLINFO_HEADER_IN:
- nbdkit_debug ("S: %s", str);
+ if (conn_id >= 0 && xfer_id >= 0)
+ nbdkit_debug ("conn %" PRIi64 " xfer %" PRIi64 ": S: %s",
+ conn_id, xfer_id, str);
+ else
+ nbdkit_debug ("S: %s", str);
break;
case CURLINFO_HEADER_OUT:
- nbdkit_debug ("C: %s", str);
+ if (conn_id >= 0 && xfer_id >= 0)
+ nbdkit_debug ("conn %" PRIi64 " xfer %" PRIi64 ": C: %s",
+ conn_id, xfer_id, str);
+ else
+ nbdkit_debug ("C: %s", str);
break;
default:
/* Assume everything else is binary data that we cannot print. */
- nbdkit_debug ("<data with size=%zu>", origsize);
+ if (conn_id >= 0 && xfer_id >= 0)
+ nbdkit_debug ("conn %" PRIi64 " xfer %" PRIi64 ": "
+ "<data with size=%zu>",
+ conn_id, xfer_id,
+ origsize);
+ else
+ nbdkit_debug ("<data with size=%zu>", origsize);
}
out:
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 1feba844..022e8c60 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -62,6 +62,10 @@
#define HAVE_CURLINFO_TOTAL_TIME_T
#define HAVE_CURLINFO_REDIRECT_TIME_T
#endif
+#if CURL_AT_LEAST_VERSION (8, 2, 0)
+#define HAVE_CURLINFO_CONN_ID
+#define HAVE_CURLINFO_XFER_ID
+#endif
#endif
extern const char *url;
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index 0774adad..7784b553 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -570,6 +570,12 @@ This enables very verbose curl debugging. See L<CURLOPT_VERBOSE(3)>.
This is mainly useful if you suspect there is a bug inside libcurl
itself.
+=item B<-D curl.verbose.ids=1>
+
+This enhances C<-D curl.verbose=1> by printing connection and transfer
+IDs next to each debug message. As this has some overhead it is not
+enabled by default.
+
=back
=head1 FILES
--
2.39.3

85
SOURCES/0018-curl-Rename-unload_config-unload_pool-config_unload-.patch
Unescape View File

@ -0,0 +1,85 @@
From ae6c568fecbef8b2aa72a89d6afd4aabcec10a05 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 14:24:04 +0100
Subject: [PATCH] curl: Rename unload_config, unload_pool -> config_unload,
pool_unload
For consistency with scripts_unload.
There is still a function called load_pool, but that inconsistency
will be removed in a follow-on commit.
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit f7bea74f670ba303c7ea60713cbbf301fab865fd)
---
plugins/curl/config.c | 2 +-
plugins/curl/curl.c | 4 ++--
plugins/curl/curldefs.h | 4 ++--
plugins/curl/pool.c | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/plugins/curl/config.c b/plugins/curl/config.c
index d68c92ba..276c79d5 100644
--- a/plugins/curl/config.c
+++ b/plugins/curl/config.c
@@ -103,7 +103,7 @@ NBDKIT_DLL_PUBLIC int curl_debug_verbose = 0;
NBDKIT_DLL_PUBLIC int curl_debug_verbose_ids = 0;
void
-unload_config (void)
+config_unload (void)
{
free (cookie);
if (headers)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 99a7e00b..72971093 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -74,9 +74,9 @@ curl_load (void)
static void
curl_unload (void)
{
- unload_config ();
+ config_unload ();
scripts_unload ();
- unload_pool ();
+ pool_unload ();
display_times ();
curl_global_cleanup ();
}
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 022e8c60..cab4a6b1 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -119,13 +119,13 @@ struct curl_handle {
extern int curl_config (const char *key, const char *value);
extern int curl_config_complete (void);
extern const char *curl_config_help;
-extern void unload_config (void);
+extern void config_unload (void);
extern struct curl_handle *allocate_handle (void);
extern void free_handle (struct curl_handle *);
/* pool.c */
extern void load_pool (void);
-extern void unload_pool (void);
+extern void pool_unload (void);
extern struct curl_handle *get_handle (void);
extern void put_handle (struct curl_handle *ch);
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 91e56f07..50a623a4 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -86,7 +86,7 @@ load_pool (void)
/* Close and free all handles in the pool. */
void
-unload_pool (void)
+pool_unload (void)
{
size_t i;
--
2.39.3

96
SOURCES/0019-pool-Add-outline-get_ready-and-after_fork-functions.patch
Unescape View File

@ -0,0 +1,96 @@
From 995b1b62add89b437cb51bb7e15a35c265e188b3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 14:33:03 +0100
Subject: [PATCH] pool: Add outline get_ready and after_fork functions
In a forthcoming commit we will need to create a multi handle and a
background thread, requiring use of the .get_ready (for multi) and
.after_fork (for the thread) plugin methods. This commit removes the
empty load_pool function and adds pool_get_ready and pool_after_fork,
and the associated machinery in curl.c.
This commit on its own does nothing, future commits will fill in these
functions with useful work.
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit 4075a499115a9e12ea74c4838fe00a7d680de2a1)
---
plugins/curl/curl.c | 14 +++++++++++++-
plugins/curl/curldefs.h | 3 ++-
plugins/curl/pool.c | 12 +++++++++---
3 files changed, 24 insertions(+), 5 deletions(-)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 72971093..4e727b86 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -67,8 +67,18 @@ curl_load (void)
nbdkit_error ("libcurl initialization failed: %d", (int) r);
exit (EXIT_FAILURE);
}
+}
- load_pool ();
+int
+curl_get_ready (void)
+{
+ return pool_get_ready ();
+}
+
+int
+curl_after_fork (void)
+{
+ return pool_after_fork ();
}
static void
@@ -249,6 +259,8 @@ static struct nbdkit_plugin plugin = {
*/
//.config_help = curl_config_help,
.magic_config_key = "url",
+ .get_ready = curl_get_ready,
+ .after_fork = curl_after_fork,
.open = curl_open,
.close = curl_close,
.get_size = curl_get_size,
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index cab4a6b1..939c8d37 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -124,7 +124,8 @@ extern struct curl_handle *allocate_handle (void);
extern void free_handle (struct curl_handle *);
/* pool.c */
-extern void load_pool (void);
+extern int pool_get_ready (void);
+extern int pool_after_fork (void);
extern void pool_unload (void);
extern struct curl_handle *get_handle (void);
extern void put_handle (struct curl_handle *ch);
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 50a623a4..eb2d330e 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -78,10 +78,16 @@ static curl_handle_list curl_handles = empty_vector;
static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
static size_t in_use = 0, waiting = 0;
-/* Initialize pool structures. */
-void
-load_pool (void)
+int
+pool_get_ready (void)
{
+ return 0;
+}
+
+int
+pool_after_fork (void)
+{
+ return 0;
}
/* Close and free all handles in the pool. */
--
2.39.3

36
SOURCES/0020-curl-Do-pool_unload-before-config_unload.patch
Unescape View File

@ -0,0 +1,36 @@
From 173530071d11433e26e1be1c11bc0e474f18079b Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 14:35:42 +0100
Subject: [PATCH] curl: Do pool_unload before config_unload
Since config.c deals with handles (and contains free_handle), and
since pool_unload calls free_handle to free handles, it's better to do
pool_unload first.
I don't believe this is a correctness issue now, but it will be in
subsequent commits.
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit 650bde3bdb85ff2af0ea618fd64e726b7535d686)
---
plugins/curl/curl.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 4e727b86..be42de36 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -84,9 +84,9 @@ curl_after_fork (void)
static void
curl_unload (void)
{
+ pool_unload ();
config_unload ();
scripts_unload ();
- pool_unload ();
display_times ();
curl_global_cleanup ();
}
--
2.39.3

52
SOURCES/0021-retry-request-Allow-get_size-operation-to-be-retried.patch
Unescape View File

@ -0,0 +1,52 @@
From 0b9807056e8be58b624b67b281accddc630c5d2c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 16:47:58 +0100
Subject: [PATCH] retry-request: Allow get_size operation to be retried
This plugin operation might need to do some real work (instead of just
fetching a number from memory), and so it might have to be retried.
In particular, changes to the curl plugin make .get_size into a
heavyweight operation, where previously it was done as a side-effect
of .open. And so we must allow .get_size to be retried independent of
.open.
(cherry picked from commit 0ff9f9e202218e278bf38faf328ec80f25d6b661)
---
filters/retry-request/retry-request.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/filters/retry-request/retry-request.c b/filters/retry-request/retry-request.c
index e5b8344c..8e3dd824 100644
--- a/filters/retry-request/retry-request.c
+++ b/filters/retry-request/retry-request.c
@@ -141,6 +141,18 @@ retry_request_open (nbdkit_next_open *next, nbdkit_context *nxdata,
return r == 0 ? NBDKIT_HANDLE_NOT_NEEDED : NULL;
}
+static int64_t
+retry_request_get_size (nbdkit_next *next, void *handle)
+{
+ int64_t r;
+ int *err = &errno; /* used by the RETRY_* macros */
+
+ RETRY_START("get_size")
+ r = next->get_size (next);
+ RETRY_END;
+ return r;
+}
+
static int
retry_request_pread (nbdkit_next *next,
void *handle, void *buf, uint32_t count, uint64_t offset,
@@ -267,6 +279,7 @@ static struct nbdkit_filter filter = {
.config = retry_request_config,
.config_help = retry_request_config_help,
.open = retry_request_open,
+ .get_size = retry_request_get_size,
.pread = retry_request_pread,
.pwrite = retry_request_pwrite,
.trim = retry_request_trim,
--
2.39.3

42
SOURCES/0022-tests-test-retry-request-mirror.c-Don-t-assume-state.patch
Unescape View File

@ -0,0 +1,42 @@
From 85241affaa52e4d67d0a22329d4cc5e2b0fe5ca3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 16:59:20 +0100
Subject: [PATCH] tests/test-retry-request-mirror.c: Don't assume state after
connect
After forthcoming changes to the curl plugin we cannot assume the
exact mirror we will be connected to after making the NBD connection.
So remove that assumption.
See: commit 38dccd848bd40cccdf012df7a606e13282aaeecb
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit ee03fbd2385dcd8772ad2c838401df0b5d8c5617)
---
tests/test-retry-request-mirror.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/tests/test-retry-request-mirror.c b/tests/test-retry-request-mirror.c
index cf42c596..65440d2e 100644
--- a/tests/test-retry-request-mirror.c
+++ b/tests/test-retry-request-mirror.c
@@ -58,7 +58,7 @@ main (int argc, char *argv[])
const char *sockpath;
CLEANUP_FREE char *usp_param = NULL;
int i, j;
- char state = 0;
+ char state;
struct nbd_handle *nbd = NULL;
#ifndef HAVE_CURLOPT_UNIX_SOCKET_PATH
@@ -105,6 +105,8 @@ main (int argc, char *argv[])
if (nbd_connect_unix (nbd, sock /* NBD socket */) == -1)
goto nbd_error;
+ state = 0;
+
for (i = 0; i < 7 /* not divisible by 2 or 3 */; ++i) {
char buf[512];
--
2.39.3

1279
SOURCES/0023-curl-Use-curl-multi-interface.patch
View File

File diff suppressed because it is too large Load Diff

139
SOURCES/0024-curl-Redefine-connections-N-parameter-as-number-of-H.patch
Unescape View File

@ -0,0 +1,139 @@
From d33cf724dccd014f854a90bb7341fd36f890bb01 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 11:49:24 +0100
Subject: [PATCH] curl: Redefine connections=<N> parameter as number of HTTP
connections
Previously (nbdkit 1.34) this was the number of easy handles. However
it turns out that easy handles can open multiple HTTP connections, and
in fact there's no good way to tell how many (and they are not
shared).
Now that we are using a curl multi, curl >= 7.30 provides a way to
limit the total number of actual HTTP connections, so we should just
use it. This is closer to what I intended this parameter to mean.
Also update the documentation to properly describe previous and
current behaviour.
Link: https://curl.se/mail/lib-2019-03/0102.html
Link: https://curl.se/mail/lib-2019-12/0044.html
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit 2bb03f898e49ec892070055b6975914642fa8c5c)
---
plugins/curl/config.c | 2 +-
plugins/curl/curldefs.h | 3 +++
plugins/curl/nbdkit-curl-plugin.pod | 37 +++++++++++++++++++----------
plugins/curl/pool.c | 6 ++++-
4 files changed, 34 insertions(+), 14 deletions(-)
diff --git a/plugins/curl/config.c b/plugins/curl/config.c
index ce82d5f9..a7501707 100644
--- a/plugins/curl/config.c
+++ b/plugins/curl/config.c
@@ -494,7 +494,7 @@ curl_config_complete (void)
const char *curl_config_help =
"cainfo=<CAINFO> Path to Certificate Authority file.\n"
"capath=<CAPATH> Path to directory with CA certificates.\n"
- "connections=<N> Number of libcurl connections to use.\n"
+ "connections=<N> Number of HTTP connections to use.\n"
"cookie=<COOKIE> Set HTTP/HTTPS cookies.\n"
"cookiefile= Enable cookie processing.\n"
"cookiefile=<FILENAME> Read cookies from file.\n"
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 6b158d85..73e7a3b4 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -50,6 +50,9 @@
* macro isn't present then Curl is very old.
*/
#ifdef CURL_AT_LEAST_VERSION
+#if CURL_AT_LEAST_VERSION (7, 30, 0)
+#define HAVE_CURLMOPT_MAX_TOTAL_CONNECTIONS
+#endif
#if CURL_AT_LEAST_VERSION (7, 55, 0)
#define HAVE_CURLINFO_CONTENT_LENGTH_DOWNLOAD_T
#endif
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index 7784b553..9821f111 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -58,10 +58,10 @@ L<CURLOPT_CAPATH(3)> for more information.
(nbdkit E<ge> 1.34)
-Open up to C<N> curl connections to the web server. The default is 4.
-Curl connections are shared between all NBD clients, so you may wish
-to increase this if you expect many simultaneous NBD clients (or a
-single client using many multi-conn connections).
+Open up to C<N> connections to the web server. The default is 16.
+Connections are shared between all NBD clients, so you may wish to
+increase this if you expect many simultaneous NBD clients (or a single
+client using many multi-conn connections).
See L</NBD CONNECTIONS AND CURL HANDLES> below.
@@ -397,15 +397,28 @@ user-agent header.
=head1 NBD CONNECTIONS AND CURL HANDLES
nbdkit E<le> 1.32 used a simple model where a new NBD connection would
-create a new libcurl handle. In practice this meant there was a
-1-to-1 relationship between NBD connections and HTTP connections to
-the remote web server (assuming http: or https: URL).
+create a new libcurl handle. Since a libcurl handle maintains a small
+cache of connections, this meant that the number of HTTP connections
+would be a small multiple of the number of incoming NBD connections
+and the total would not be limited (assuming http: or https: URL).
-nbdkit E<ge> 1.34 changed to using a fixed pool of libcurl handles
-shared across all NBD connections. You can control the maximum number
-of curl handles in the pool with the C<connections> parameter (default
-4). Note that if there are more than 4 NBD connections, they will
-share the 4 web server connections, unless you adjust C<connections>.
+nbdkit 1.34 changed to using a fixed pool of libcurl handles shared
+across all NBD connections. You can control the maximum number of
+curl handles in the pool with the C<connections> parameter (default
+4). Since each curl handle maintains a small cache of connections,
+this meant that the number of HTTP connections would be a small
+multiple of the C<connections> parameter. If there are more than 4
+incoming NBD connections, they will contend for the libcurl handles,
+unless you adjust C<connections>.
+
+nbdkit E<ge> 1.36 changed again to use a curl multi handle
+(L<libcurl-multi(3)>). Now the C<connections> parameter controls the
+maximum number of HTTP connections made to the remote server
+(L<CURLMOPT_MAX_TOTAL_CONNECTIONS(3)>). This is more efficient
+especially with HTTP/2 and HTTP/3, where each HTTP connection can
+contain a very large number of streams (typically up to 100)
+multiplexed over one connection. The default for C<connections> was
+raised to 16.
=head1 HEADER AND COOKIE SCRIPTS
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 254951d1..7d44dfe5 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -79,7 +79,7 @@
/* Use '-D curl.pool=1' to debug handle pool. */
NBDKIT_DLL_PUBLIC int curl_debug_pool = 0;
-unsigned connections = 4;
+unsigned connections = 16;
/* Pipe used to notify background thread that a command is pending in
* the queue. A pointer to the 'struct command' is sent over the
@@ -115,6 +115,10 @@ pool_get_ready (void)
return -1;
}
+#ifdef HAVE_CURLMOPT_MAX_TOTAL_CONNECTIONS
+ curl_multi_setopt(multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) connections);
+#endif
+
return 0;
}
--
2.39.3

57
SOURCES/0025-curl-Disable-this-plugin-on-Windows.patch
Unescape View File

@ -0,0 +1,57 @@
From 852f23db007b13ca8e9e5548c3becbdc78e16601 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 1 Aug 2023 11:29:04 +0100
Subject: [PATCH] curl: Disable this plugin on Windows
There is no self pipe trick for Windows (or there is, but it would
require substantial porting work).
(cherry picked from commit 9b0759377f6779bbecc8647c026dcdac7f2ebd89)
---
plugins/curl/Makefile.am | 5 ++++-
tests/Makefile.am | 2 ++
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/plugins/curl/Makefile.am b/plugins/curl/Makefile.am
index 7529fb2f..971365f1 100644
--- a/plugins/curl/Makefile.am
+++ b/plugins/curl/Makefile.am
@@ -34,6 +34,8 @@ include $(top_srcdir)/common-rules.mk
EXTRA_DIST = nbdkit-curl-plugin.pod
if HAVE_CURL
+# Disabled on Windows because no self-pipe.
+if !IS_WINDOWS
plugin_LTLIBRARIES = nbdkit-curl-plugin.la
@@ -85,4 +87,5 @@ nbdkit-curl-plugin.1: nbdkit-curl-plugin.pod \
endif HAVE_POD
-endif
+endif !IS_WINDOWS
+endif HAVE_CURL
diff --git a/tests/Makefile.am b/tests/Makefile.am
index f6c5ac9a..08514c7a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -643,6 +643,7 @@ EXTRA_DIST += test-cdi.sh
# curl plugin test.
if HAVE_MKE2FS_WITH_D
if HAVE_CURL
+if !IS_WINDOWS
TESTS += \
test-curl-file.sh \
test-curl-header-script-fail.sh \
@@ -749,6 +750,7 @@ test_curl_cookie_script_LDADD = \
$(LIBNBD_LIBS) \
$(NULL)
+endif !IS_WINDOWS
endif HAVE_CURL
endif HAVE_MKE2FS_WITH_D
--
2.39.3

2
SOURCES/copy-patches.sh
Unescape View File

@ -6,7 +6,7 @@ set -e
# directory. Use it like this:
# ./copy-patches.sh
rhel_version=10.0
rhel_version=9.3
# Check we're in the right directory.
if [ ! -f nbdkit.spec ]; then

17
SOURCES/nbdkit-1.34.2.tar.gz.sig
Unescape View File

@ -0,0 +1,17 @@
-----BEGIN PGP SIGNATURE-----
iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmS2eroRHHJpY2hAYW5u
ZXhpYS5vcmcACgkQkXOPc+G3aKBhcg/+IMU4RsKeuCiU6Y5k+bAYDN1bZWdGowM5
kRiarFOaO+917qj2K1FohXqz8xv+j7OSgQ0AyAfHEYCcrvwUoS3nxld/xf6Vq2NW
myPR4PdhKXx+ed2Y1PRwUykzdHLKIbV2itIQMvZJK6lN8CBEwooFOCbaJuZBUTxX
RA7seFbQeW5sSvqwfAEEPOtvj9Hj4AWOhbYHnxtzIH9NN+skkXATPR8ieqn/64Qx
JVa4Wx23nEHpfR0TzWdIbx08ggcmZstlVu1eiUB8MFhIdPnMnW3hPPWTcScZ45YP
kuxbqk5Q+AG4MsaGIiXaHryRW+bzr/Nn5Wb3AU+As2ZYtLRi4rvcGSPy1dTjblnU
5xVMz2bWjd0g/yU7ErD3NFGhp/RWfCOSOZ3AawbI8FdCMCWcJNIWNtT2sR7Ukqft
vOvim7oulUH6oLWwS/IkH9/SSxRXObMbnZTcqADaCCtif3F+9RkbbHeNfE85UEqm
Qc6VZoea2c2K/GTzfRDxoGuexU8p2zjiGiLltoMQboJqIU/Bz6q5Nqm8agL7Lm4x
/9HXUoF1+OQ8Ga18bQQbwrYIDcsB/xuMq3SlKo5h7az/XMeg3Pxz+YuBIMOcOBP7
N8M/yQZJIs5jkKkJaEt9gX0SzLAK0AiwtMlupew4B9iWrF+uUvM9z2pcxdqDJMCO
EOl+wAKKYqo=
=yvkW
-----END PGP SIGNATURE-----

17
SOURCES/nbdkit-1.40.0.tar.gz.sig
Unescape View File

@ -1,17 +0,0 @@
-----BEGIN PGP SIGNATURE-----
iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmaddOcRHHJpY2hAYW5u
ZXhpYS5vcmcACgkQkXOPc+G3aKCyHQ//QSHYXixx8DJLlpRHhUYzcKOLSez+rvkN
TjwTfzIKxQXYbke8pqt3izwEIDTmMCeIVDUAjXj3bpnlVAx8I0hE+DHQe1AlbIXx
VjgmdTVcT/v5nL39DVqMXxVmXQwCtgcOlwQuZw6YSsdyPnH/UeplyrOjGN5W/XE7
KPmfU1slBx//ybLp4L4qQCmTfDLqBLzVRTGyR7UJDrle/J5jinqmNzLcAvKI0WM0
Incv4CvNj85mYtE2nWiBofwCK1OyiNvsWhsEuRxu/8OIGtPVrho4WuAemlVDbabL
JsHgZPwGECDewPEdFnfZJhrvhvgzYkDjsXjqLC0aXX61efDbnTPkkLduJlPMqhMa
obgS5xX5gIdwJFnVOX5yO2OVd5ghGDG+pi+yqt6fL0QPqzDBW6leupri4sBI/C3J
dE7o//i2MUDYeFWGEpIMsY0X/JIDILU76DKDsM0C9WsMfKe5sOJ968iFFjVR7EEP
cxwSqhSJrMV3kUEBNoTb6g6md2Gld/0iQ85hhCw8BeBnADU+tiRmasLvYT2+uCUz
fiwld0QRfgWq/UkjmDCvbfYypb5p/F/Wu6Z6tqPFOl3+eJ0adp7C41fqSGjar7ZH
9yelDZ1UpHr85em+cZ+FGt1UvTtlYVM2+EcWLVLZ8NpcxziZBmwBw3upIKC3CYTu
6K4aw/YrmE0=
=oWs8
-----END PGP SIGNATURE-----

3
SOURCES/nbdkit.fc
Unescape View File

@ -1,3 +0,0 @@
/usr/sbin/nbdkit -- gen_context(system_u:object_r:nbdkit_exec_t,s0)
/usr/lib/systemd/system/nbdkit.* gen_context(system_u:object_r:nbdkit_unit_file_t,s0)

207
SOURCES/nbdkit.if
Unescape View File

@ -1,207 +0,0 @@
## <summary>policy for nbdkit</summary>
########################################
## <summary>
## Execute nbdkit_exec_t in the nbdkit domain.
## </summary>
## <param name="domain">
## <summary>
## Domain allowed to transition.
## </summary>
## </param>
#
interface(`nbdkit_domtrans',`
gen_require(`
type nbdkit_t, nbdkit_exec_t;
')
corecmd_search_bin($1)
domtrans_pattern($1, nbdkit_exec_t, nbdkit_t)
')
######################################
## <summary>
## Execute nbdkit in the caller domain.
## </summary>
## <param name="domain">
## <summary>
## Domain allowed access.
## </summary>
## </param>
#
interface(`nbdkit_exec',`
gen_require(`
type nbdkit_exec_t;
')
corecmd_search_bin($1)
can_exec($1, nbdkit_exec_t)
')
########################################
## <summary>
## Execute nbdkit in the nbdkit domain, and
## allow the specified role the nbdkit domain.
## </summary>
## <param name="domain">
## <summary>
## Domain allowed to transition
## </summary>
## </param>
## <param name="role">
## <summary>
## The role to be allowed the nbdkit domain.
## </summary>
## </param>
#
interface(`nbdkit_run',`
gen_require(`
type nbdkit_t;
attribute_role nbdkit_roles;
')
nbdkit_domtrans($1)
roleattribute $2 nbdkit_roles;
')
########################################
## <summary>
## Role access for nbdkit
## </summary>
## <param name="role">
## <summary>
## Role allowed access
## </summary>
## </param>
## <param name="domain">
## <summary>
## User domain for the role
## </summary>
## </param>
#
interface(`nbdkit_role',`
gen_require(`
type nbdkit_t;
attribute_role nbdkit_roles;
')
roleattribute $1 nbdkit_roles;
nbdkit_domtrans($2)
ps_process_pattern($2, nbdkit_t)
allow $2 nbdkit_t:process { signull signal sigkill };
')
########################################
## <summary>
## Allow attempts to connect to nbdkit
## with a unix stream socket.
## </summary>
## <param name="domain">
## <summary>
## Domain to not audit.
## </summary>
## </param>
#
interface(`nbdkit_stream_connect',`
gen_require(`
type nbdkit_t;
')
allow $1 nbdkit_t:unix_stream_socket connectto;
')
########################################
## <summary>
## Allow nbdkit_exec_t to be an entrypoint
## of the specified domain
## </summary>
## <param name="domain">
## <summary>
## Domain allowed access.
## </summary>
## </param>
## <rolecap/>
#
interface(`nbdkit_entrypoint',`
gen_require(`
type nbdkit_exec_t;
')
allow $1 nbdkit_exec_t:file entrypoint;
')
# ----------------------------------------------------------------------
# RWMJ: See:
# https://issues.redhat.com/browse/RHEL-5174?focusedId=23387259&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-23387259
# Remove this when virt.if gets updated.
########################################
#
# Interface compatibility blocks
#
# The following definitions ensure compatibility with distribution policy
# versions that do not contain given interfaces (epel, or older Fedora
# releases).
# Each block tests for existence of given interface and defines it if needed.
#
########################################
## <summary>
## Read and write to svirt_image dirs.
## </summary>
## <param name="domain">
## <summary>
## Domain allowed access.
## </summary>
## </param>
#
ifndef(`virt_rw_svirt_image_dirs',`
interface(`virt_rw_svirt_image_dirs',`
gen_require(`
type svirt_image_t;
')
allow $1 svirt_image_t:dir rw_dir_perms;
')
')
########################################
## <summary>
## Create svirt_image sock_files.
## </summary>
## <param name="domain">
## <summary>
## Domain allowed access.
## </summary>
## </param>
#
ifndef(`virt_create_svirt_image_sock_files',`
interface(`virt_create_svirt_image_sock_files',`
gen_require(`
type svirt_image_t;
')
allow $1 svirt_image_t:sock_file create_sock_file_perms;
')
')
########################################
## <summary>
## Read and write virtlogd pipes.
## </summary>
## <param name="domain">
## <summary>
## Domain allowed access.
## </summary>
## </param>
#
ifndef(`virtlogd_rw_pipes',`
interface(`virtlogd_rw_pipes',`
gen_require(`
type virtlogd_t;
')
allow $1 virtlogd_t:fifo_file rw_fifo_file_perms;
')
')

100
SOURCES/nbdkit.te
Unescape View File

@ -1,100 +0,0 @@
policy_module(nbdkit, 1.0.0)
########################################
#
# Declarations
#
gen_require(`
type unconfined_t;
')
type nbdkit_t;
type nbdkit_exec_t;
application_domain(nbdkit_t, nbdkit_exec_t)
mcs_constrained(nbdkit_t)
role system_r types nbdkit_t;
type nbdkit_home_t;
userdom_user_home_content(nbdkit_home_t)
type nbdkit_tmp_t;
files_tmp_file(nbdkit_tmp_t)
type nbdkit_unit_file_t;
systemd_unit_file(nbdkit_unit_file_t)
permissive nbdkit_t;
########################################
#
# nbdkit local policy
#
allow nbdkit_t self:capability { setgid setuid };
allow nbdkit_t self:fifo_file rw_fifo_file_perms;
allow nbdkit_t self:netlink_route_socket rw_netlink_socket_perms;
allow nbdkit_t self:process { fork setsockcreate signal_perms };
allow nbdkit_t self:tcp_socket create_stream_socket_perms;
allow nbdkit_t self:udp_socket create_socket_perms;
manage_dirs_pattern(nbdkit_t, nbdkit_tmp_t, nbdkit_tmp_t)
manage_files_pattern(nbdkit_t, nbdkit_tmp_t, nbdkit_tmp_t)
userdom_user_tmp_filetrans(nbdkit_t, nbdkit_tmp_t, { dir file })
manage_dirs_pattern(nbdkit_t, nbdkit_home_t, nbdkit_home_t)
manage_files_pattern(nbdkit_t, nbdkit_home_t, nbdkit_home_t)
userdom_user_home_dir_filetrans(nbdkit_t, nbdkit_home_t, { dir file })
corenet_tcp_connect_http_port(nbdkit_t)
corenet_tcp_connect_ssh_port(nbdkit_t)
corenet_tcp_connect_tftp_port(nbdkit_t)
corenet_tcp_bind_generic_port(nbdkit_t)
corenet_tcp_bind_generic_node(nbdkit_t)
domain_use_interactive_fds(nbdkit_t)
files_read_etc_files(nbdkit_t)
init_abstract_socket_activation(nbdkit_t)
init_ioctl_stream_sockets(nbdkit_t)
init_rw_stream_sockets(nbdkit_t)
optional_policy(`
auth_use_nsswitch(nbdkit_t)
')
optional_policy(`
logging_send_syslog_msg(nbdkit_t)
')
optional_policy(`
miscfiles_read_localization(nbdkit_t)
miscfiles_read_generic_certs(nbdkit_t)
')
optional_policy(`
sysnet_dns_name_resolve(nbdkit_t)
sysnet_read_config(nbdkit_t)
')
optional_policy(`
userdom_read_user_home_content_files(nbdkit_t)
userdom_use_inherited_user_ptys(nbdkit_t)
')
optional_policy(`
virt_create_svirt_image_sock_files(nbdkit_t)
virt_read_qemu_pid_files(nbdkit_t)
virtlogd_rw_pipes(nbdkit_t)
virt_rw_svirt_image(nbdkit_t)
virt_rw_svirt_image_dirs(nbdkit_t)
virt_search_lib(nbdkit_t)
virt_stream_connect_svirt(nbdkit_t)
')
# FIXME: It would be nice to allow libvirt to transition nbdkit_exec_t to
# nbdkit_t when libvirtd was started manually from the commandline (i.e. in
# unconfined_t), but we don't want this transition to happen automatically
# when starting directly from the shell. I'm not sure how to achieve this...
#nbdkit_domtrans(unconfined_t, nbdkit_exec_t, nbdkit_t)

1078
SPECS/nbdkit.spec
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save