parent
6d0b712af0
commit
43af285c5d
@ -1,2 +1,2 @@
|
|||||||
SOURCES/libguestfs.keyring
|
SOURCES/libguestfs.keyring
|
||||||
SOURCES/libnbd-1.12.6.tar.gz
|
SOURCES/libnbd-1.14.2.tar.gz
|
||||||
|
@ -1,2 +1,2 @@
|
|||||||
cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring
|
cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring
|
||||||
2d4eb0846d51c25fa7d04295972cbb5a617984ef SOURCES/libnbd-1.12.6.tar.gz
|
6bc66366a216117c3c451dc7764c790435749b80 SOURCES/libnbd-1.14.2.tar.gz
|
||||||
|
File diff suppressed because it is too large
Load Diff
@ -1,153 +0,0 @@
|
|||||||
From ec947323528725fcf12b5b9ba32b02d36dbd9621 Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
||||||
Date: Thu, 30 Jun 2022 21:09:39 +0100
|
|
||||||
Subject: [PATCH] dump: Visually separate columns 0-7 and 8-15
|
|
||||||
MIME-Version: 1.0
|
|
||||||
Content-Type: text/plain; charset=UTF-8
|
|
||||||
Content-Transfer-Encoding: 8bit
|
|
||||||
|
|
||||||
Before:
|
|
||||||
|
|
||||||
0000090000: 68 65 72 65 20 77 65 20 61 72 65 00 68 65 72 65 │...
|
|
||||||
0000090010: 20 77 65 20 61 72 65 00 68 65 72 65 20 77 65 20 │...
|
|
||||||
0000090020: 61 72 65 00 68 65 72 65 20 77 65 20 61 72 65 00 │...
|
|
||||||
|
|
||||||
After:
|
|
||||||
|
|
||||||
0000090000: 68 65 72 65 20 77 65 20 61 72 65 00 68 65 72 65 │...
|
|
||||||
0000090010: 20 77 65 20 61 72 65 00 68 65 72 65 20 77 65 20 │...
|
|
||||||
0000090020: 61 72 65 00 68 65 72 65 20 77 65 20 61 72 65 00 │...
|
|
||||||
|
|
||||||
Updates: commit c4107b9a40d6451630dcccf1bf6596c8e56420be
|
|
||||||
(cherry picked from commit 315a637d3eae003c1d84eb1b88a7b47b534f1e80)
|
|
||||||
---
|
|
||||||
dump/dump-data.sh | 22 +++++++++++-----------
|
|
||||||
dump/dump-empty-qcow2.sh | 4 ++--
|
|
||||||
dump/dump-pattern.sh | 38 +++++++++++++++++++-------------------
|
|
||||||
dump/dump.c | 5 ++++-
|
|
||||||
4 files changed, 36 insertions(+), 33 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/dump/dump-data.sh b/dump/dump-data.sh
|
|
||||||
index 23d09da..955cd3b 100755
|
|
||||||
--- a/dump/dump-data.sh
|
|
||||||
+++ b/dump/dump-data.sh
|
|
||||||
@@ -37,21 +37,21 @@ nbdkit -U - data data='
|
|
||||||
|
|
||||||
cat $output
|
|
||||||
|
|
||||||
-if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
+if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
*
|
|
||||||
-0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
-0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
+0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
+0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
*
|
|
||||||
-000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h|
|
|
||||||
-0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....|
|
|
||||||
-0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
+000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h|
|
|
||||||
+0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....|
|
|
||||||
+0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
*
|
|
||||||
-00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp|
|
|
||||||
-0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo|
|
|
||||||
-0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........|
|
|
||||||
-0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
+00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp|
|
|
||||||
+0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo|
|
|
||||||
+0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........|
|
|
||||||
+0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
*
|
|
||||||
-0001312d00: 00 |. |' ]; then
|
|
||||||
+0001312d00: 00 |. |' ]; then
|
|
||||||
echo "$0: unexpected output from nbddump command"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
diff --git a/dump/dump-empty-qcow2.sh b/dump/dump-empty-qcow2.sh
|
|
||||||
index c9e583b..472b6eb 100755
|
|
||||||
--- a/dump/dump-empty-qcow2.sh
|
|
||||||
+++ b/dump/dump-empty-qcow2.sh
|
|
||||||
@@ -38,9 +38,9 @@ qemu-img create -f qcow2 $file $size
|
|
||||||
nbddump -- [ $QEMU_NBD -r -f qcow2 $file ] > $output
|
|
||||||
cat $output
|
|
||||||
|
|
||||||
-if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
+if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
|
|
||||||
*
|
|
||||||
-003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then
|
|
||||||
+003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then
|
|
||||||
echo "$0: unexpected output from nbddump command"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh
|
|
||||||
index e4016a8..d512b77 100755
|
|
||||||
--- a/dump/dump-pattern.sh
|
|
||||||
+++ b/dump/dump-pattern.sh
|
|
||||||
@@ -32,25 +32,25 @@ nbdkit -U - pattern size=299 --run 'nbddump "$uri"' > $output
|
|
||||||
|
|
||||||
cat $output
|
|
||||||
|
|
||||||
-if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................|
|
|
||||||
-0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................|
|
|
||||||
-0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(|
|
|
||||||
-0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8|
|
|
||||||
-0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H|
|
|
||||||
-0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X|
|
|
||||||
-0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h|
|
|
||||||
-0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x|
|
|
||||||
-0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................|
|
|
||||||
-0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................|
|
|
||||||
-00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................|
|
|
||||||
-00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................|
|
|
||||||
-00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................|
|
|
||||||
-00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................|
|
|
||||||
-00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................|
|
|
||||||
-00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................|
|
|
||||||
-0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................|
|
|
||||||
-0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................|
|
|
||||||
-0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then
|
|
||||||
+if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................|
|
|
||||||
+0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................|
|
|
||||||
+0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(|
|
|
||||||
+0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8|
|
|
||||||
+0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H|
|
|
||||||
+0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X|
|
|
||||||
+0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h|
|
|
||||||
+0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x|
|
|
||||||
+0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................|
|
|
||||||
+0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................|
|
|
||||||
+00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................|
|
|
||||||
+00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................|
|
|
||||||
+00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................|
|
|
||||||
+00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................|
|
|
||||||
+00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................|
|
|
||||||
+00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................|
|
|
||||||
+0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................|
|
|
||||||
+0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................|
|
|
||||||
+0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then
|
|
||||||
echo "$0: unexpected output from nbddump command"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
diff --git a/dump/dump.c b/dump/dump.c
|
|
||||||
index 76af04c..7818f1f 100644
|
|
||||||
--- a/dump/dump.c
|
|
||||||
+++ b/dump/dump.c
|
|
||||||
@@ -429,10 +429,13 @@ do_dump (void)
|
|
||||||
else
|
|
||||||
ansi_grey ();
|
|
||||||
printf ("%02x ", buffer[j]);
|
|
||||||
+ if ((j - i) == 7) printf (" ");
|
|
||||||
}
|
|
||||||
ansi_grey ();
|
|
||||||
- for (; j < i+16; ++j)
|
|
||||||
+ for (; j < i+16; ++j) {
|
|
||||||
printf (" ");
|
|
||||||
+ if ((j - i) == 7) printf (" ");
|
|
||||||
+ }
|
|
||||||
|
|
||||||
/* Print the ASCII codes. */
|
|
||||||
printf ("%s", pipe);
|
|
||||||
--
|
|
||||||
2.31.1
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
|||||||
From e7a2815412891d5c13b5b5f0e9aa61882880c87f Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
||||||
Date: Thu, 30 Jun 2022 22:31:00 +0100
|
|
||||||
Subject: [PATCH] dump: Fix tests on Debian 10
|
|
||||||
|
|
||||||
The version of nbdkit on Debian 10 does not set $uri. Check for this
|
|
||||||
or skip the test.
|
|
||||||
|
|
||||||
(cherry picked from commit 083b1ca30fb5e6e0dc0e4b0eea9ebe8474d3f864)
|
|
||||||
---
|
|
||||||
dump/dump-data.sh | 1 +
|
|
||||||
dump/dump-pattern.sh | 1 +
|
|
||||||
2 files changed, 2 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/dump/dump-data.sh b/dump/dump-data.sh
|
|
||||||
index 955cd3b..46e4d1e 100755
|
|
||||||
--- a/dump/dump-data.sh
|
|
||||||
+++ b/dump/dump-data.sh
|
|
||||||
@@ -23,6 +23,7 @@ set -x
|
|
||||||
|
|
||||||
requires nbdkit --version
|
|
||||||
requires nbdkit data --dump-plugin
|
|
||||||
+requires nbdkit -U - null --run 'test "$uri" != ""'
|
|
||||||
|
|
||||||
output=dump-data.out
|
|
||||||
rm -f $output
|
|
||||||
diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh
|
|
||||||
index d512b77..e2188ac 100755
|
|
||||||
--- a/dump/dump-pattern.sh
|
|
||||||
+++ b/dump/dump-pattern.sh
|
|
||||||
@@ -23,6 +23,7 @@ set -x
|
|
||||||
|
|
||||||
requires nbdkit --version
|
|
||||||
requires nbdkit pattern --dump-plugin
|
|
||||||
+requires nbdkit -U - null --run 'test "$uri" != ""'
|
|
||||||
|
|
||||||
output=dump-pattern.out
|
|
||||||
rm -f $output
|
|
||||||
--
|
|
||||||
2.31.1
|
|
||||||
|
|
@ -1,31 +0,0 @@
|
|||||||
From 7c669783b1b3fab902ce34d7914b62617ed8b263 Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
||||||
Date: Thu, 30 Jun 2022 22:35:05 +0100
|
|
||||||
Subject: [PATCH] dump/dump-data.sh: Test requires nbdkit 1.22
|
|
||||||
|
|
||||||
Ubuntu 20.04 has nbdkit 1.16 which lacks support for strings. These
|
|
||||||
were added in nbdkit 1.22.
|
|
||||||
|
|
||||||
(cherry picked from commit a8fa05ffb8b85f41276ffb52498e4528c08e5f21)
|
|
||||||
---
|
|
||||||
dump/dump-data.sh | 4 ++++
|
|
||||||
1 file changed, 4 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/dump/dump-data.sh b/dump/dump-data.sh
|
|
||||||
index 46e4d1e..11145b0 100755
|
|
||||||
--- a/dump/dump-data.sh
|
|
||||||
+++ b/dump/dump-data.sh
|
|
||||||
@@ -25,6 +25,10 @@ requires nbdkit --version
|
|
||||||
requires nbdkit data --dump-plugin
|
|
||||||
requires nbdkit -U - null --run 'test "$uri" != ""'
|
|
||||||
|
|
||||||
+# This test requires nbdkit >= 1.22.
|
|
||||||
+minor=$( nbdkit --dump-config | grep ^version_minor | cut -d= -f2 )
|
|
||||||
+requires test $minor -ge 22
|
|
||||||
+
|
|
||||||
output=dump-data.out
|
|
||||||
rm -f $output
|
|
||||||
cleanup_fn rm -f $output
|
|
||||||
--
|
|
||||||
2.31.1
|
|
||||||
|
|
@ -1,163 +0,0 @@
|
|||||||
From 8dce43a3ea7a529bc37cbe5607a8d52186cc8169 Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
||||||
Date: Tue, 28 Jun 2022 18:27:58 +0100
|
|
||||||
Subject: [PATCH] copy: Store the preferred block size in the operations struct
|
|
||||||
|
|
||||||
This will be used in a subsequent commit. At the moment the preferred
|
|
||||||
block size for all sources / destinations is simply calculated and
|
|
||||||
stored.
|
|
||||||
|
|
||||||
(cherry picked from commit e6c42f8b2d447bbcc659d6dd33be67335834b2e5)
|
|
||||||
---
|
|
||||||
copy/file-ops.c | 4 +++-
|
|
||||||
copy/main.c | 29 +++++++++++++++++++++++------
|
|
||||||
copy/nbd-ops.c | 10 ++++++++++
|
|
||||||
copy/nbdcopy.h | 4 +++-
|
|
||||||
copy/null-ops.c | 1 +
|
|
||||||
copy/pipe-ops.c | 1 +
|
|
||||||
6 files changed, 41 insertions(+), 8 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/copy/file-ops.c b/copy/file-ops.c
|
|
||||||
index ab37875..34f08e5 100644
|
|
||||||
--- a/copy/file-ops.c
|
|
||||||
+++ b/copy/file-ops.c
|
|
||||||
@@ -241,13 +241,15 @@ seek_hole_supported (int fd)
|
|
||||||
|
|
||||||
struct rw *
|
|
||||||
file_create (const char *name, int fd,
|
|
||||||
- off_t st_size, bool is_block, direction d)
|
|
||||||
+ off_t st_size, uint64_t preferred,
|
|
||||||
+ bool is_block, direction d)
|
|
||||||
{
|
|
||||||
struct rw_file *rwf = calloc (1, sizeof *rwf);
|
|
||||||
if (rwf == NULL) { perror ("calloc"); exit (EXIT_FAILURE); }
|
|
||||||
|
|
||||||
rwf->rw.ops = &file_ops;
|
|
||||||
rwf->rw.name = name;
|
|
||||||
+ rwf->rw.preferred = preferred;
|
|
||||||
rwf->fd = fd;
|
|
||||||
rwf->is_block = is_block;
|
|
||||||
|
|
||||||
diff --git a/copy/main.c b/copy/main.c
|
|
||||||
index cc379e9..19ec384 100644
|
|
||||||
--- a/copy/main.c
|
|
||||||
+++ b/copy/main.c
|
|
||||||
@@ -512,10 +512,26 @@ open_local (const char *filename, direction d)
|
|
||||||
fprintf (stderr, "%s: %s: %m\n", prog, filename);
|
|
||||||
exit (EXIT_FAILURE);
|
|
||||||
}
|
|
||||||
- if (S_ISBLK (stat.st_mode) || S_ISREG (stat.st_mode))
|
|
||||||
- return file_create (filename, fd, stat.st_size, S_ISBLK (stat.st_mode), d);
|
|
||||||
- else {
|
|
||||||
- /* Probably stdin/stdout, a pipe or a socket. */
|
|
||||||
+ if (S_ISREG (stat.st_mode)) /* Regular file. */
|
|
||||||
+ return file_create (filename, fd,
|
|
||||||
+ stat.st_size, (uint64_t) stat.st_blksize, false, d);
|
|
||||||
+ else if (S_ISBLK (stat.st_mode)) { /* Block device. */
|
|
||||||
+ unsigned int blkioopt;
|
|
||||||
+
|
|
||||||
+#ifdef BLKIOOPT
|
|
||||||
+ if (ioctl (fd, BLKIOOPT, &blkioopt) == -1) {
|
|
||||||
+ fprintf (stderr, "warning: cannot get optimal I/O size: %s: %m",
|
|
||||||
+ filename);
|
|
||||||
+ blkioopt = 4096;
|
|
||||||
+ }
|
|
||||||
+#else
|
|
||||||
+ blkioopt = 4096;
|
|
||||||
+#endif
|
|
||||||
+
|
|
||||||
+ return file_create (filename, fd,
|
|
||||||
+ stat.st_size, (uint64_t) blkioopt, true, d);
|
|
||||||
+ }
|
|
||||||
+ else { /* Probably stdin/stdout, a pipe or a socket. */
|
|
||||||
synchronous = true; /* Force synchronous mode for pipes. */
|
|
||||||
return pipe_create (filename, fd);
|
|
||||||
}
|
|
||||||
@@ -528,8 +544,9 @@ print_rw (struct rw *rw, const char *prefix, FILE *fp)
|
|
||||||
char buf[HUMAN_SIZE_LONGEST];
|
|
||||||
|
|
||||||
fprintf (fp, "%s: %s \"%s\"\n", prefix, rw->ops->ops_name, rw->name);
|
|
||||||
- fprintf (fp, "%s: size=%" PRIi64 " (%s)\n",
|
|
||||||
- prefix, rw->size, human_size (buf, rw->size, NULL));
|
|
||||||
+ fprintf (fp, "%s: size=%" PRIi64 " (%s), preferred block size=%" PRIu64 "\n",
|
|
||||||
+ prefix, rw->size, human_size (buf, rw->size, NULL),
|
|
||||||
+ rw->preferred);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Default implementation of rw->ops->get_extents for backends which
|
|
||||||
diff --git a/copy/nbd-ops.c b/copy/nbd-ops.c
|
|
||||||
index 3bc26ba..0988634 100644
|
|
||||||
--- a/copy/nbd-ops.c
|
|
||||||
+++ b/copy/nbd-ops.c
|
|
||||||
@@ -112,12 +112,22 @@ open_one_nbd_handle (struct rw_nbd *rwn)
|
|
||||||
* the same way.
|
|
||||||
*/
|
|
||||||
if (rwn->handles.len == 0) {
|
|
||||||
+ int64_t block_size;
|
|
||||||
+
|
|
||||||
rwn->can_zero = nbd_can_zero (nbd) > 0;
|
|
||||||
+
|
|
||||||
rwn->rw.size = nbd_get_size (nbd);
|
|
||||||
if (rwn->rw.size == -1) {
|
|
||||||
fprintf (stderr, "%s: %s: %s\n", prog, rwn->rw.name, nbd_get_error ());
|
|
||||||
exit (EXIT_FAILURE);
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ block_size = nbd_get_block_size (nbd, LIBNBD_SIZE_PREFERRED);
|
|
||||||
+ if (block_size == -1) {
|
|
||||||
+ fprintf (stderr, "%s: %s: %s\n", prog, rwn->rw.name, nbd_get_error ());
|
|
||||||
+ exit (EXIT_FAILURE);
|
|
||||||
+ }
|
|
||||||
+ rwn->rw.preferred = block_size == 0 ? 4096 : block_size;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (handles_append (&rwn->handles, nbd) == -1) {
|
|
||||||
diff --git a/copy/nbdcopy.h b/copy/nbdcopy.h
|
|
||||||
index 19797df..9438cce 100644
|
|
||||||
--- a/copy/nbdcopy.h
|
|
||||||
+++ b/copy/nbdcopy.h
|
|
||||||
@@ -43,6 +43,7 @@ struct rw {
|
|
||||||
struct rw_ops *ops; /* Operations. */
|
|
||||||
const char *name; /* Printable name, for error messages etc. */
|
|
||||||
int64_t size; /* May be -1 for streams. */
|
|
||||||
+ uint64_t preferred; /* Preferred block size. */
|
|
||||||
/* Followed by private data for the particular subtype. */
|
|
||||||
};
|
|
||||||
|
|
||||||
@@ -53,7 +54,8 @@ typedef enum { READING, WRITING } direction;
|
|
||||||
|
|
||||||
/* Create subtypes. */
|
|
||||||
extern struct rw *file_create (const char *name, int fd,
|
|
||||||
- off_t st_size, bool is_block, direction d);
|
|
||||||
+ off_t st_size, uint64_t preferred,
|
|
||||||
+ bool is_block, direction d);
|
|
||||||
extern struct rw *nbd_rw_create_uri (const char *name,
|
|
||||||
const char *uri, direction d);
|
|
||||||
extern struct rw *nbd_rw_create_subprocess (const char **argv, size_t argc,
|
|
||||||
diff --git a/copy/null-ops.c b/copy/null-ops.c
|
|
||||||
index 1218a62..99cc9a7 100644
|
|
||||||
--- a/copy/null-ops.c
|
|
||||||
+++ b/copy/null-ops.c
|
|
||||||
@@ -45,6 +45,7 @@ null_create (const char *name)
|
|
||||||
rw->rw.ops = &null_ops;
|
|
||||||
rw->rw.name = name;
|
|
||||||
rw->rw.size = INT64_MAX;
|
|
||||||
+ rw->rw.preferred = 4096;
|
|
||||||
return &rw->rw;
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/copy/pipe-ops.c b/copy/pipe-ops.c
|
|
||||||
index 3c8b6c2..3815f82 100644
|
|
||||||
--- a/copy/pipe-ops.c
|
|
||||||
+++ b/copy/pipe-ops.c
|
|
||||||
@@ -43,6 +43,7 @@ pipe_create (const char *name, int fd)
|
|
||||||
rwp->rw.ops = &pipe_ops;
|
|
||||||
rwp->rw.name = name;
|
|
||||||
rwp->rw.size = -1;
|
|
||||||
+ rwp->rw.preferred = 4096;
|
|
||||||
rwp->fd = fd;
|
|
||||||
return &rwp->rw;
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.31.1
|
|
||||||
|
|
@ -1,457 +0,0 @@
|
|||||||
From c8626acc63c4ae1c6cf5d1505e0209ac10f44e81 Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
||||||
Date: Tue, 28 Jun 2022 21:58:55 +0100
|
|
||||||
Subject: [PATCH] copy: Use preferred block size for copying
|
|
||||||
|
|
||||||
You're not supposed to read or write NBD servers at a granularity less
|
|
||||||
than the advertised minimum block size. nbdcopy has ignored this
|
|
||||||
requirement, and this is usually fine because the NBD servers we care
|
|
||||||
about support 512-byte sector granularity, and never advertise sizes /
|
|
||||||
extents less granular than sectors (even if it's a bit suboptimal in a
|
|
||||||
few cases).
|
|
||||||
|
|
||||||
However there is one new case where we do care: When writing to a
|
|
||||||
compressed qcow2 file, qemu advertises a minimum and preferred block
|
|
||||||
size of 64K, and it really means it. You cannot write blocks smaller
|
|
||||||
than this because of the way qcow2 compression is implemented.
|
|
||||||
|
|
||||||
This commit attempts to do the least work possible to fix this.
|
|
||||||
|
|
||||||
The previous multi-thread-copying loop was driven by the extent map
|
|
||||||
received from the source. I have modified the loop so that it
|
|
||||||
iterates over request_size blocks. request_size is set from the
|
|
||||||
command line (--request-size) but will be adjusted upwards if either
|
|
||||||
the source or destination preferred block size is larger. So this
|
|
||||||
will always copy blocks which are at least the preferred block size
|
|
||||||
(except for the very last block of the disk).
|
|
||||||
|
|
||||||
While copying these blocks we consult the source extent map. If it
|
|
||||||
contains only zero regions covering the whole block (only_zeroes
|
|
||||||
function) then we can skip straight to zeroing the target
|
|
||||||
(fill_dst_range_with_zeroes), else we do read + write as before.
|
|
||||||
|
|
||||||
I only modified the multi-thread-copying loop, not the synchronous
|
|
||||||
loop. That should be updated in the same way later.
|
|
||||||
|
|
||||||
One side effect of this change is it always makes larger requests,
|
|
||||||
even for regions we know are sparse. This is clear in the
|
|
||||||
copy-sparse.sh and copy-sparse-allocated.sh tests which were
|
|
||||||
previously driven by the 32K sparse map granularity of the source.
|
|
||||||
Without changing these tests, they would make make 256K reads & writes
|
|
||||||
(and also read from areas of the disk even though we know they are
|
|
||||||
sparse). I adjusted these tests to use --request-size=32768 to force
|
|
||||||
the existing behaviour.
|
|
||||||
|
|
||||||
Note this doesn't attempt to limit the maximum block size when reading
|
|
||||||
or writing. That is for future work.
|
|
||||||
|
|
||||||
This is a partial fix for https://bugzilla.redhat.com/2047660.
|
|
||||||
Further changes will be required in virt-v2v.
|
|
||||||
|
|
||||||
Link: https://lists.gnu.org/archive/html/qemu-block/2022-01/threads.html#00729
|
|
||||||
Link: https://bugzilla.redhat.com/show_bug.cgi?id=2047660
|
|
||||||
(cherry picked from commit 4058fe1ff03fb41156b67302ba1006b9d06b0218)
|
|
||||||
---
|
|
||||||
TODO | 4 +-
|
|
||||||
copy/Makefile.am | 6 +-
|
|
||||||
copy/copy-file-to-qcow2-compressed.sh | 64 +++++++++++
|
|
||||||
copy/copy-sparse-allocated.sh | 4 +-
|
|
||||||
copy/copy-sparse.sh | 7 +-
|
|
||||||
copy/main.c | 13 +++
|
|
||||||
copy/multi-thread-copying.c | 149 +++++++++++++++++++-------
|
|
||||||
copy/nbdcopy.pod | 5 +-
|
|
||||||
8 files changed, 202 insertions(+), 50 deletions(-)
|
|
||||||
create mode 100755 copy/copy-file-to-qcow2-compressed.sh
|
|
||||||
|
|
||||||
diff --git a/TODO b/TODO
|
|
||||||
index 7c9c15e..bc38d70 100644
|
|
||||||
--- a/TODO
|
|
||||||
+++ b/TODO
|
|
||||||
@@ -28,7 +28,9 @@ Performance: Chart it over various buffer sizes and threads, as that
|
|
||||||
Examine other fuzzers: https://gitlab.com/akihe/radamsa
|
|
||||||
|
|
||||||
nbdcopy:
|
|
||||||
- - Minimum/preferred/maximum block size.
|
|
||||||
+ - Enforce maximum block size.
|
|
||||||
+ - Synchronous loop should be adjusted to take into account
|
|
||||||
+ the NBD preferred block size, as was done for multi-thread loop.
|
|
||||||
- Benchmark.
|
|
||||||
- Better page cache usage, see nbdkit-file-plugin options
|
|
||||||
fadvise=sequential cache=none.
|
|
||||||
diff --git a/copy/Makefile.am b/copy/Makefile.am
|
|
||||||
index e729f86..25f75c5 100644
|
|
||||||
--- a/copy/Makefile.am
|
|
||||||
+++ b/copy/Makefile.am
|
|
||||||
@@ -23,6 +23,7 @@ EXTRA_DIST = \
|
|
||||||
copy-file-to-nbd.sh \
|
|
||||||
copy-file-to-null.sh \
|
|
||||||
copy-file-to-qcow2.sh \
|
|
||||||
+ copy-file-to-qcow2-compressed.sh \
|
|
||||||
copy-nbd-to-block.sh \
|
|
||||||
copy-nbd-to-file.sh \
|
|
||||||
copy-nbd-to-hexdump.sh \
|
|
||||||
@@ -142,7 +143,10 @@ TESTS += \
|
|
||||||
$(NULL)
|
|
||||||
|
|
||||||
if HAVE_QEMU_NBD
|
|
||||||
-TESTS += copy-file-to-qcow2.sh
|
|
||||||
+TESTS += \
|
|
||||||
+ copy-file-to-qcow2.sh \
|
|
||||||
+ copy-file-to-qcow2-compressed.sh \
|
|
||||||
+ $(NULL)
|
|
||||||
endif
|
|
||||||
|
|
||||||
if HAVE_GNUTLS
|
|
||||||
diff --git a/copy/copy-file-to-qcow2-compressed.sh b/copy/copy-file-to-qcow2-compressed.sh
|
|
||||||
new file mode 100755
|
|
||||||
index 0000000..dfe4fa5
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/copy/copy-file-to-qcow2-compressed.sh
|
|
||||||
@@ -0,0 +1,64 @@
|
|
||||||
+#!/usr/bin/env bash
|
|
||||||
+# nbd client library in userspace
|
|
||||||
+# Copyright (C) 2020-2022 Red Hat Inc.
|
|
||||||
+#
|
|
||||||
+# This library is free software; you can redistribute it and/or
|
|
||||||
+# modify it under the terms of the GNU Lesser General Public
|
|
||||||
+# License as published by the Free Software Foundation; either
|
|
||||||
+# version 2 of the License, or (at your option) any later version.
|
|
||||||
+#
|
|
||||||
+# This library is distributed in the hope that it will be useful,
|
|
||||||
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
||||||
+# Lesser General Public License for more details.
|
|
||||||
+#
|
|
||||||
+# You should have received a copy of the GNU Lesser General Public
|
|
||||||
+# License along with this library; if not, write to the Free Software
|
|
||||||
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
||||||
+
|
|
||||||
+. ../tests/functions.sh
|
|
||||||
+
|
|
||||||
+set -e
|
|
||||||
+set -x
|
|
||||||
+
|
|
||||||
+requires $QEMU_NBD --version
|
|
||||||
+requires nbdkit --exit-with-parent --version
|
|
||||||
+requires nbdkit sparse-random --dump-plugin
|
|
||||||
+requires qemu-img --version
|
|
||||||
+requires stat --version
|
|
||||||
+
|
|
||||||
+file1=copy-file-to-qcow2-compressed.file1
|
|
||||||
+file2=copy-file-to-qcow2-compressed.file2
|
|
||||||
+rm -f $file1 $file2
|
|
||||||
+cleanup_fn rm -f $file1 $file2
|
|
||||||
+
|
|
||||||
+size=1G
|
|
||||||
+seed=$RANDOM
|
|
||||||
+
|
|
||||||
+# Create a compressed qcow2 file1.
|
|
||||||
+#
|
|
||||||
+# sparse-random files should compress easily because by default each
|
|
||||||
+# block uses repeated bytes.
|
|
||||||
+qemu-img create -f qcow2 $file1 $size
|
|
||||||
+nbdcopy -- [ nbdkit --exit-with-parent sparse-random $size seed=$seed ] \
|
|
||||||
+ [ $QEMU_NBD --image-opts driver=compress,file.driver=qcow2,file.file.driver=file,file.file.filename=$file1 ]
|
|
||||||
+
|
|
||||||
+ls -l $file1
|
|
||||||
+
|
|
||||||
+# Create an uncompressed qcow2 file2 with the same data.
|
|
||||||
+qemu-img create -f qcow2 $file2 $size
|
|
||||||
+nbdcopy -- [ nbdkit --exit-with-parent sparse-random $size seed=$seed ] \
|
|
||||||
+ [ $QEMU_NBD --image-opts driver=qcow2,file.driver=file,file.filename=$file2 ]
|
|
||||||
+
|
|
||||||
+ls -l $file2
|
|
||||||
+
|
|
||||||
+# file1 < file2 (shows the compression is having some effect).
|
|
||||||
+size1="$( stat -c %s $file1 )"
|
|
||||||
+size2="$( stat -c %s $file2 )"
|
|
||||||
+if [ $size1 -ge $size2 ]; then
|
|
||||||
+ echo "$0: qcow2 compression did not make the file smaller"
|
|
||||||
+ exit 1
|
|
||||||
+fi
|
|
||||||
+
|
|
||||||
+# Logical content of the files should be identical.
|
|
||||||
+qemu-img compare -f qcow2 $file1 -F qcow2 $file2
|
|
||||||
diff --git a/copy/copy-sparse-allocated.sh b/copy/copy-sparse-allocated.sh
|
|
||||||
index 203c3b9..465e347 100755
|
|
||||||
--- a/copy/copy-sparse-allocated.sh
|
|
||||||
+++ b/copy/copy-sparse-allocated.sh
|
|
||||||
@@ -17,8 +17,6 @@
|
|
||||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
||||||
|
|
||||||
# Adapted from copy-sparse.sh.
|
|
||||||
-#
|
|
||||||
-# This test depends on the nbdkit default sparse block size (32K).
|
|
||||||
|
|
||||||
. ../tests/functions.sh
|
|
||||||
|
|
||||||
@@ -33,7 +31,7 @@ requires nbdkit eval --version
|
|
||||||
out=copy-sparse-allocated.out
|
|
||||||
cleanup_fn rm -f $out
|
|
||||||
|
|
||||||
-$VG nbdcopy --allocated -- \
|
|
||||||
+$VG nbdcopy --allocated --request-size=32768 -- \
|
|
||||||
[ nbdkit --exit-with-parent data data='
|
|
||||||
1
|
|
||||||
@1073741823 1
|
|
||||||
diff --git a/copy/copy-sparse.sh b/copy/copy-sparse.sh
|
|
||||||
index 1a6da86..7912a21 100755
|
|
||||||
--- a/copy/copy-sparse.sh
|
|
||||||
+++ b/copy/copy-sparse.sh
|
|
||||||
@@ -16,8 +16,6 @@
|
|
||||||
# License along with this library; if not, write to the Free Software
|
|
||||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
||||||
|
|
||||||
-# This test depends on the nbdkit default sparse block size (32K).
|
|
||||||
-
|
|
||||||
. ../tests/functions.sh
|
|
||||||
|
|
||||||
set -e
|
|
||||||
@@ -34,8 +32,9 @@ cleanup_fn rm -f $out
|
|
||||||
# Copy from a sparse data disk to an nbdkit-eval-plugin instance which
|
|
||||||
# is logging everything. This allows us to see exactly what nbdcopy
|
|
||||||
# is writing, to ensure it is writing and zeroing the target as
|
|
||||||
-# expected.
|
|
||||||
-$VG nbdcopy -S 0 -- \
|
|
||||||
+# expected. Force request size to match nbdkit default sparse
|
|
||||||
+# allocator block size (32K).
|
|
||||||
+$VG nbdcopy -S 0 --request-size=32768 -- \
|
|
||||||
[ nbdkit --exit-with-parent data data='
|
|
||||||
1
|
|
||||||
@1073741823 1
|
|
||||||
diff --git a/copy/main.c b/copy/main.c
|
|
||||||
index 19ec384..0e27db8 100644
|
|
||||||
--- a/copy/main.c
|
|
||||||
+++ b/copy/main.c
|
|
||||||
@@ -40,6 +40,7 @@
|
|
||||||
|
|
||||||
#include "ispowerof2.h"
|
|
||||||
#include "human-size.h"
|
|
||||||
+#include "minmax.h"
|
|
||||||
#include "version.h"
|
|
||||||
#include "nbdcopy.h"
|
|
||||||
|
|
||||||
@@ -379,10 +380,22 @@ main (int argc, char *argv[])
|
|
||||||
if (threads < connections)
|
|
||||||
connections = threads;
|
|
||||||
|
|
||||||
+ /* request_size must always be at least as large as the preferred
|
|
||||||
+ * size of source & destination.
|
|
||||||
+ */
|
|
||||||
+ request_size = MAX (request_size, src->preferred);
|
|
||||||
+ request_size = MAX (request_size, dst->preferred);
|
|
||||||
+
|
|
||||||
/* Adapt queue to size to request size if needed. */
|
|
||||||
if (request_size > queue_size)
|
|
||||||
queue_size = request_size;
|
|
||||||
|
|
||||||
+ /* Sparse size (if using) must not be smaller than the destination
|
|
||||||
+ * preferred size, otherwise we end up creating too small requests.
|
|
||||||
+ */
|
|
||||||
+ if (sparse_size > 0 && sparse_size < dst->preferred)
|
|
||||||
+ sparse_size = dst->preferred;
|
|
||||||
+
|
|
||||||
/* Truncate the destination to the same size as the source. Only
|
|
||||||
* has an effect on regular files.
|
|
||||||
*/
|
|
||||||
diff --git a/copy/multi-thread-copying.c b/copy/multi-thread-copying.c
|
|
||||||
index 06cdb8e..9267545 100644
|
|
||||||
--- a/copy/multi-thread-copying.c
|
|
||||||
+++ b/copy/multi-thread-copying.c
|
|
||||||
@@ -166,6 +166,62 @@ decrease_queue_size (struct worker *worker, size_t len)
|
|
||||||
worker->queue_size -= len;
|
|
||||||
}
|
|
||||||
|
|
||||||
+/* Using the extents map 'exts', check if the region
|
|
||||||
+ * [offset..offset+len-1] intersects only with zero extents.
|
|
||||||
+ *
|
|
||||||
+ * The invariant for '*i' is always an extent which starts before or
|
|
||||||
+ * equal to the current offset.
|
|
||||||
+ */
|
|
||||||
+static bool
|
|
||||||
+only_zeroes (const extent_list exts, size_t *i,
|
|
||||||
+ uint64_t offset, unsigned len)
|
|
||||||
+{
|
|
||||||
+ size_t j;
|
|
||||||
+
|
|
||||||
+ /* Invariant. */
|
|
||||||
+ assert (*i < exts.len);
|
|
||||||
+ assert (exts.ptr[*i].offset <= offset);
|
|
||||||
+
|
|
||||||
+ /* Update the invariant. Search for the last possible extent in the
|
|
||||||
+ * list which is <= offset.
|
|
||||||
+ */
|
|
||||||
+ for (j = *i + 1; j < exts.len; ++j) {
|
|
||||||
+ if (exts.ptr[j].offset <= offset)
|
|
||||||
+ *i = j;
|
|
||||||
+ else
|
|
||||||
+ break;
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ /* Check invariant again. */
|
|
||||||
+ assert (*i < exts.len);
|
|
||||||
+ assert (exts.ptr[*i].offset <= offset);
|
|
||||||
+
|
|
||||||
+ /* If *i is not the last extent, then the next extent starts
|
|
||||||
+ * strictly beyond our current offset.
|
|
||||||
+ */
|
|
||||||
+ assert (*i == exts.len - 1 || exts.ptr[*i + 1].offset > offset);
|
|
||||||
+
|
|
||||||
+ /* Search forward, look for any non-zero extents overlapping the region. */
|
|
||||||
+ for (j = *i; j < exts.len; ++j) {
|
|
||||||
+ uint64_t start, end;
|
|
||||||
+
|
|
||||||
+ /* [start..end-1] is the current extent. */
|
|
||||||
+ start = exts.ptr[j].offset;
|
|
||||||
+ end = exts.ptr[j].offset + exts.ptr[j].length;
|
|
||||||
+
|
|
||||||
+ assert (end > offset);
|
|
||||||
+
|
|
||||||
+ if (start >= offset + len)
|
|
||||||
+ break;
|
|
||||||
+
|
|
||||||
+ /* Non-zero extent covering this region => test failed. */
|
|
||||||
+ if (!exts.ptr[j].zero)
|
|
||||||
+ return false;
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return true;
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
/* There are 'threads' worker threads, each copying work ranges from
|
|
||||||
* src to dst until there are no more work ranges.
|
|
||||||
*/
|
|
||||||
@@ -177,7 +233,10 @@ worker_thread (void *wp)
|
|
||||||
extent_list exts = empty_vector;
|
|
||||||
|
|
||||||
while (get_next_offset (&offset, &count)) {
|
|
||||||
- size_t i;
|
|
||||||
+ struct command *command;
|
|
||||||
+ size_t extent_index;
|
|
||||||
+ bool is_zeroing = false;
|
|
||||||
+ uint64_t zeroing_start = 0; /* initialized to avoid bogus GCC warning */
|
|
||||||
|
|
||||||
assert (0 < count && count <= THREAD_WORK_SIZE);
|
|
||||||
if (extents)
|
|
||||||
@@ -185,52 +244,64 @@ worker_thread (void *wp)
|
|
||||||
else
|
|
||||||
default_get_extents (src, w->index, offset, count, &exts);
|
|
||||||
|
|
||||||
- for (i = 0; i < exts.len; ++i) {
|
|
||||||
- struct command *command;
|
|
||||||
- size_t len;
|
|
||||||
+ extent_index = 0; // index into extents array used to optimize only_zeroes
|
|
||||||
+ while (count) {
|
|
||||||
+ const size_t len = MIN (count, request_size);
|
|
||||||
|
|
||||||
- if (exts.ptr[i].zero) {
|
|
||||||
+ if (only_zeroes (exts, &extent_index, offset, len)) {
|
|
||||||
/* The source is zero so we can proceed directly to skipping,
|
|
||||||
- * fast zeroing, or writing zeroes at the destination.
|
|
||||||
+ * fast zeroing, or writing zeroes at the destination. Defer
|
|
||||||
+ * zeroing so we can send it as a single large command.
|
|
||||||
*/
|
|
||||||
- command = create_command (exts.ptr[i].offset, exts.ptr[i].length,
|
|
||||||
- true, w);
|
|
||||||
- fill_dst_range_with_zeroes (command);
|
|
||||||
+ if (!is_zeroing) {
|
|
||||||
+ is_zeroing = true;
|
|
||||||
+ zeroing_start = offset;
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
-
|
|
||||||
else /* data */ {
|
|
||||||
- /* As the extent might be larger than permitted for a single
|
|
||||||
- * command, we may have to split this into multiple read
|
|
||||||
- * requests.
|
|
||||||
- */
|
|
||||||
- while (exts.ptr[i].length > 0) {
|
|
||||||
- len = exts.ptr[i].length;
|
|
||||||
- if (len > request_size)
|
|
||||||
- len = request_size;
|
|
||||||
-
|
|
||||||
- command = create_command (exts.ptr[i].offset, len,
|
|
||||||
- false, w);
|
|
||||||
-
|
|
||||||
- wait_for_request_slots (w);
|
|
||||||
-
|
|
||||||
- /* NOTE: Must increase the queue size after waiting. */
|
|
||||||
- increase_queue_size (w, len);
|
|
||||||
-
|
|
||||||
- /* Begin the asynch read operation. */
|
|
||||||
- src->ops->asynch_read (src, command,
|
|
||||||
- (nbd_completion_callback) {
|
|
||||||
- .callback = finished_read,
|
|
||||||
- .user_data = command,
|
|
||||||
- });
|
|
||||||
-
|
|
||||||
- exts.ptr[i].offset += len;
|
|
||||||
- exts.ptr[i].length -= len;
|
|
||||||
+ /* If we were in the middle of deferred zeroing, do it now. */
|
|
||||||
+ if (is_zeroing) {
|
|
||||||
+ /* Note that offset-zeroing_start can never exceed
|
|
||||||
+ * THREAD_WORK_SIZE, so there is no danger of overflowing
|
|
||||||
+ * size_t.
|
|
||||||
+ */
|
|
||||||
+ command = create_command (zeroing_start, offset-zeroing_start,
|
|
||||||
+ true, w);
|
|
||||||
+ fill_dst_range_with_zeroes (command);
|
|
||||||
+ is_zeroing = false;
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ /* Issue the asynchronous read command. */
|
|
||||||
+ command = create_command (offset, len, false, w);
|
|
||||||
+
|
|
||||||
+ wait_for_request_slots (w);
|
|
||||||
+
|
|
||||||
+ /* NOTE: Must increase the queue size after waiting. */
|
|
||||||
+ increase_queue_size (w, len);
|
|
||||||
+
|
|
||||||
+ /* Begin the asynch read operation. */
|
|
||||||
+ src->ops->asynch_read (src, command,
|
|
||||||
+ (nbd_completion_callback) {
|
|
||||||
+ .callback = finished_read,
|
|
||||||
+ .user_data = command,
|
|
||||||
+ });
|
|
||||||
}
|
|
||||||
|
|
||||||
- offset += count;
|
|
||||||
- count = 0;
|
|
||||||
- } /* for extents */
|
|
||||||
+ offset += len;
|
|
||||||
+ count -= len;
|
|
||||||
+ } /* while (count) */
|
|
||||||
+
|
|
||||||
+ /* If we were in the middle of deferred zeroing, do it now. */
|
|
||||||
+ if (is_zeroing) {
|
|
||||||
+ /* Note that offset-zeroing_start can never exceed
|
|
||||||
+ * THREAD_WORK_SIZE, so there is no danger of overflowing
|
|
||||||
+ * size_t.
|
|
||||||
+ */
|
|
||||||
+ command = create_command (zeroing_start, offset - zeroing_start,
|
|
||||||
+ true, w);
|
|
||||||
+ fill_dst_range_with_zeroes (command);
|
|
||||||
+ is_zeroing = false;
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Wait for in flight NBD requests to finish. */
|
|
||||||
diff --git a/copy/nbdcopy.pod b/copy/nbdcopy.pod
|
|
||||||
index fd10f7c..f06d112 100644
|
|
||||||
--- a/copy/nbdcopy.pod
|
|
||||||
+++ b/copy/nbdcopy.pod
|
|
||||||
@@ -182,8 +182,9 @@ Set the maximum number of requests in flight per NBD connection.
|
|
||||||
=item B<--sparse=>N
|
|
||||||
|
|
||||||
Detect all zero blocks of size N (bytes) and make them sparse on the
|
|
||||||
-output. You can also turn off sparse detection using S<I<-S 0>>.
|
|
||||||
-The default is 4096 bytes.
|
|
||||||
+output. You can also turn off sparse detection using S<I<-S 0>>. The
|
|
||||||
+default is 4096 bytes, or the destination preferred block size,
|
|
||||||
+whichever is larger.
|
|
||||||
|
|
||||||
=item B<--synchronous>
|
|
||||||
|
|
||||||
--
|
|
||||||
2.31.1
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
|||||||
From 5d21b00dbdd1e1a04317bf16afb8f4d2ceaa470f Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
||||||
Date: Sat, 2 Jul 2022 17:12:46 +0100
|
|
||||||
Subject: [PATCH] dump: Add another example to the manual
|
|
||||||
|
|
||||||
(cherry picked from commit be3768b077c9542aba34eb821016c36f31d234af)
|
|
||||||
---
|
|
||||||
dump/nbddump.pod | 5 +++++
|
|
||||||
1 file changed, 5 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/dump/nbddump.pod b/dump/nbddump.pod
|
|
||||||
index 5d7864d..656a965 100644
|
|
||||||
--- a/dump/nbddump.pod
|
|
||||||
+++ b/dump/nbddump.pod
|
|
||||||
@@ -57,6 +57,11 @@ For example, to dump out a qcow2 file as raw data:
|
|
||||||
|
|
||||||
nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ]
|
|
||||||
|
|
||||||
+To dump out an empty floppy disk created by L<nbdkit-floppy-plugin(1)>:
|
|
||||||
+
|
|
||||||
+ mkdir /var/tmp/empty
|
|
||||||
+ nbddump -- [ nbdkit floppy /var/tmp/empty ]
|
|
||||||
+
|
|
||||||
Note that S<C<[ ... ]>> are separate parameters, and must be
|
|
||||||
surrounded by spaces. C<--> separates nbddump parameters from
|
|
||||||
subprocess parameters.
|
|
||||||
--
|
|
||||||
2.31.1
|
|
||||||
|
|
@ -1,93 +0,0 @@
|
|||||||
From a432e773e0cdc24cb27ccdda4111744ea2c3b819 Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
||||||
Date: Wed, 27 Jul 2022 17:08:14 +0100
|
|
||||||
Subject: [PATCH] lib/crypto: Use GNUTLS_NO_SIGNAL if available
|
|
||||||
|
|
||||||
libnbd has long used MSG_NOSIGNAL to avoid receiving SIGPIPE if we
|
|
||||||
accidentally write on a closed socket, which is a nice alternative to
|
|
||||||
using a SIGPIPE signal handler. However with TLS connections, gnutls
|
|
||||||
did not use this flag and so programs using libnbd + TLS would receive
|
|
||||||
SIGPIPE in some situations, notably if the server closed the
|
|
||||||
connection abruptly while we were trying to write something.
|
|
||||||
|
|
||||||
GnuTLS 3.4.2 introduces GNUTLS_NO_SIGNAL which does the same thing.
|
|
||||||
Use this flag if available.
|
|
||||||
|
|
||||||
RHEL 7 has an older gnutls which lacks this flag. To avoid qemu-nbd
|
|
||||||
interop tests failing (rarely, but more often with a forthcoming
|
|
||||||
change to TLS shutdown behaviour), register a SIGPIPE signal handler
|
|
||||||
in the test if the flag is missing.
|
|
||||||
---
|
|
||||||
configure.ac | 15 +++++++++++++++
|
|
||||||
interop/interop.c | 10 ++++++++++
|
|
||||||
lib/crypto.c | 7 ++++++-
|
|
||||||
3 files changed, 31 insertions(+), 1 deletion(-)
|
|
||||||
|
|
||||||
diff --git a/configure.ac b/configure.ac
|
|
||||||
index 49ca8ab..6bd9e1b 100644
|
|
||||||
--- a/configure.ac
|
|
||||||
+++ b/configure.ac
|
|
||||||
@@ -179,6 +179,21 @@ AS_IF([test "$GNUTLS_LIBS" != ""],[
|
|
||||||
gnutls_session_set_verify_cert \
|
|
||||||
gnutls_transport_is_ktls_enabled \
|
|
||||||
])
|
|
||||||
+ AC_MSG_CHECKING([if gnutls has GNUTLS_NO_SIGNAL])
|
|
||||||
+ AC_COMPILE_IFELSE(
|
|
||||||
+ [AC_LANG_PROGRAM([
|
|
||||||
+ #include <gnutls/gnutls.h>
|
|
||||||
+ gnutls_session_t session;
|
|
||||||
+ ], [
|
|
||||||
+ gnutls_init(&session, GNUTLS_CLIENT|GNUTLS_NO_SIGNAL);
|
|
||||||
+ ])
|
|
||||||
+ ], [
|
|
||||||
+ AC_MSG_RESULT([yes])
|
|
||||||
+ AC_DEFINE([HAVE_GNUTLS_NO_SIGNAL], [1],
|
|
||||||
+ [GNUTLS_NO_SIGNAL found at compile time])
|
|
||||||
+ ], [
|
|
||||||
+ AC_MSG_RESULT([no])
|
|
||||||
+ ])
|
|
||||||
LIBS="$old_LIBS"
|
|
||||||
])
|
|
||||||
|
|
||||||
diff --git a/interop/interop.c b/interop/interop.c
|
|
||||||
index b41f3ca..036545b 100644
|
|
||||||
--- a/interop/interop.c
|
|
||||||
+++ b/interop/interop.c
|
|
||||||
@@ -84,6 +84,16 @@ main (int argc, char *argv[])
|
|
||||||
REQUIRES
|
|
||||||
#endif
|
|
||||||
|
|
||||||
+ /* Ignore SIGPIPE. We only need this for GnuTLS < 3.4.2, since
|
|
||||||
+ * newer GnuTLS has the GNUTLS_NO_SIGNAL flag which adds
|
|
||||||
+ * MSG_NOSIGNAL to each write call.
|
|
||||||
+ */
|
|
||||||
+#if !HAVE_GNUTLS_NO_SIGNAL
|
|
||||||
+#if TLS
|
|
||||||
+ signal (SIGPIPE, SIG_IGN);
|
|
||||||
+#endif
|
|
||||||
+#endif
|
|
||||||
+
|
|
||||||
/* Create a large sparse temporary file. */
|
|
||||||
#ifdef NEEDS_TMPFILE
|
|
||||||
int fd = mkstemp (TMPFILE);
|
|
||||||
diff --git a/lib/crypto.c b/lib/crypto.c
|
|
||||||
index 1272888..ca9520e 100644
|
|
||||||
--- a/lib/crypto.c
|
|
||||||
+++ b/lib/crypto.c
|
|
||||||
@@ -588,7 +588,12 @@ nbd_internal_crypto_create_session (struct nbd_handle *h,
|
|
||||||
gnutls_psk_client_credentials_t pskcreds = NULL;
|
|
||||||
gnutls_certificate_credentials_t xcreds = NULL;
|
|
||||||
|
|
||||||
- err = gnutls_init (&session, GNUTLS_CLIENT|GNUTLS_NONBLOCK);
|
|
||||||
+ err = gnutls_init (&session,
|
|
||||||
+ GNUTLS_CLIENT | GNUTLS_NONBLOCK
|
|
||||||
+#if HAVE_GNUTLS_NO_SIGNAL
|
|
||||||
+ | GNUTLS_NO_SIGNAL
|
|
||||||
+#endif
|
|
||||||
+ );
|
|
||||||
if (err < 0) {
|
|
||||||
set_error (errno, "gnutls_init: %s", gnutls_strerror (err));
|
|
||||||
return NULL;
|
|
||||||
--
|
|
||||||
2.31.1
|
|
||||||
|
|
@ -1,100 +0,0 @@
|
|||||||
From 8bbee9c0ff052cf8ab5ba81fd1b67e3c45e7012a Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
||||||
Date: Wed, 27 Jul 2022 16:07:37 +0100
|
|
||||||
Subject: [PATCH] lib/crypto.c: Ignore TLS premature termination after write
|
|
||||||
shutdown
|
|
||||||
|
|
||||||
qemu-nbd doesn't call gnutls_bye to cleanly shut down the connection
|
|
||||||
after we send NBD_CMD_DISC. When copying from a qemu-nbd server (or
|
|
||||||
any operation which calls nbd_shutdown) you will see errors like this:
|
|
||||||
|
|
||||||
$ nbdcopy nbds://foo?tls-certificates=/var/tmp/pki null:
|
|
||||||
nbds://foo?tls-certificates=/var/tmp/pki: nbd_shutdown: gnutls_record_recv: The TLS connection was non-properly terminated.
|
|
||||||
|
|
||||||
Relatedly you may also see:
|
|
||||||
|
|
||||||
nbd_shutdown: gnutls_record_recv: Error in the pull function.
|
|
||||||
|
|
||||||
This commit suppresses the error in the case where we know that we
|
|
||||||
have shut down writes (which happens after NBD_CMD_DISC has been sent
|
|
||||||
on the wire).
|
|
||||||
---
|
|
||||||
interop/interop.c | 9 ---------
|
|
||||||
lib/crypto.c | 17 +++++++++++++++++
|
|
||||||
lib/internal.h | 1 +
|
|
||||||
3 files changed, 18 insertions(+), 9 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/interop/interop.c b/interop/interop.c
|
|
||||||
index 036545b..cce9407 100644
|
|
||||||
--- a/interop/interop.c
|
|
||||||
+++ b/interop/interop.c
|
|
||||||
@@ -226,19 +226,10 @@ main (int argc, char *argv[])
|
|
||||||
|
|
||||||
/* XXX In future test more operations here. */
|
|
||||||
|
|
||||||
-#if !TLS
|
|
||||||
- /* XXX qemu doesn't shut down the connection nicely (using
|
|
||||||
- * gnutls_bye) and because of this the following call will fail
|
|
||||||
- * with:
|
|
||||||
- *
|
|
||||||
- * nbd_shutdown: gnutls_record_recv: The TLS connection was
|
|
||||||
- * non-properly terminated.
|
|
||||||
- */
|
|
||||||
if (nbd_shutdown (nbd, 0) == -1) {
|
|
||||||
fprintf (stderr, "%s\n", nbd_get_error ());
|
|
||||||
exit (EXIT_FAILURE);
|
|
||||||
}
|
|
||||||
-#endif
|
|
||||||
|
|
||||||
nbd_close (nbd);
|
|
||||||
|
|
||||||
diff --git a/lib/crypto.c b/lib/crypto.c
|
|
||||||
index ca9520e..aa5d820 100644
|
|
||||||
--- a/lib/crypto.c
|
|
||||||
+++ b/lib/crypto.c
|
|
||||||
@@ -187,6 +187,22 @@ tls_recv (struct nbd_handle *h, struct socket *sock, void *buf, size_t len)
|
|
||||||
errno = EAGAIN;
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
+ if (h->tls_shut_writes &&
|
|
||||||
+ (r == GNUTLS_E_PULL_ERROR || r == GNUTLS_E_PREMATURE_TERMINATION)) {
|
|
||||||
+ /* qemu-nbd doesn't call gnutls_bye to cleanly shut down the
|
|
||||||
+ * connection after we send NBD_CMD_DISC, instead it simply
|
|
||||||
+ * closes the connection. On the client side we see
|
|
||||||
+ * "gnutls_record_recv: The TLS connection was non-properly
|
|
||||||
+ * terminated" or "gnutls_record_recv: Error in the pull
|
|
||||||
+ * function.".
|
|
||||||
+ *
|
|
||||||
+ * If we see these errors after we shut down the write side
|
|
||||||
+ * (h->tls_shut_writes), which happens after we have sent
|
|
||||||
+ * NBD_CMD_DISC on the wire, downgrade them to a debug message.
|
|
||||||
+ */
|
|
||||||
+ debug (h, "gnutls_record_recv: %s", gnutls_strerror (r));
|
|
||||||
+ return 0; /* EOF */
|
|
||||||
+ }
|
|
||||||
set_error (0, "gnutls_record_recv: %s", gnutls_strerror (r));
|
|
||||||
errno = EIO;
|
|
||||||
return -1;
|
|
||||||
@@ -234,6 +250,7 @@ tls_shut_writes (struct nbd_handle *h, struct socket *sock)
|
|
||||||
return false;
|
|
||||||
if (r != 0)
|
|
||||||
debug (h, "ignoring gnutls_bye failure: %s", gnutls_strerror (r));
|
|
||||||
+ h->tls_shut_writes = true;
|
|
||||||
return sock->u.tls.oldsock->ops->shut_writes (h, sock->u.tls.oldsock);
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/lib/internal.h b/lib/internal.h
|
|
||||||
index 6aaced3..f1b4c63 100644
|
|
||||||
--- a/lib/internal.h
|
|
||||||
+++ b/lib/internal.h
|
|
||||||
@@ -307,6 +307,7 @@ struct nbd_handle {
|
|
||||||
struct command *reply_cmd;
|
|
||||||
|
|
||||||
bool disconnect_request; /* True if we've queued NBD_CMD_DISC */
|
|
||||||
+ bool tls_shut_writes; /* Used by lib/crypto.c to track disconnect. */
|
|
||||||
};
|
|
||||||
|
|
||||||
struct meta_context {
|
|
||||||
--
|
|
||||||
2.31.1
|
|
||||||
|
|
@ -1,17 +0,0 @@
|
|||||||
-----BEGIN PGP SIGNATURE-----
|
|
||||||
|
|
||||||
iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmLhNK4RHHJpY2hAYW5u
|
|
||||||
ZXhpYS5vcmcACgkQkXOPc+G3aKDzmxAArFrR/cOyqyGZXuYORFRVi7AobCjum4dP
|
|
||||||
A93R43shnSXXB1CTww5O+LjIghSLs4TEQAOcmcsjsE98X2cz0BuW6gIfGxTpN3WP
|
|
||||||
fGPDlvezLXGo5zX5WGFkP6oQY97TuGHXKNxStZtWRtDNfrWPWJQuwlm5GSIHdYYr
|
|
||||||
dFssmDNtIoh/zQz2www9JKspMfehFbTGZswtRjfDwa2Pl69cMy3pH/k4EZZnDx9n
|
|
||||||
tguzQHOapJJx8RkIwUwFirCBOwdVNbLX+KrGroLcB6MjO6Uhh2C/iYUQDM/xX2r6
|
|
||||||
1SugssAmXwZ4/RIDtwBLdQdEoNjSAV7OW1yizTl5P9qFkiPr+Lpnpt4gci9bTZKK
|
|
||||||
oIy0RtgJOgW5R2tuRlMXkx/7kcGjhUb3Zbux7d7dgrYq16FUPC1dFgub7WSPSqWe
|
|
||||||
+17iUD+n3NO2MHtR215nKDjuPR59wnvATO6QS+InOb4imyf47Ic7TeEvVuDhb+M3
|
|
||||||
+DvIor2WyXWX9kO165Fx9jAicgZJt2L1UvKM1GzGjwBdL0GYbCFltzzhyi4Pd/7x
|
|
||||||
ijV2QYbyOLXYEpsgmKYrT6MwwXUAye2PkXSv8MaOs3IiFz0bVy3Bfgx4UYNbKo1x
|
|
||||||
zwVGtIBz39tXpgyS7+F9rvQILVmENGmyTx+GXrv/lE1mFmTt/EEyf+iHMSN1lIkt
|
|
||||||
59o9LBpOajI=
|
|
||||||
=idnt
|
|
||||||
-----END PGP SIGNATURE-----
|
|
@ -0,0 +1,17 @@
|
|||||||
|
-----BEGIN PGP SIGNATURE-----
|
||||||
|
|
||||||
|
iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmO0NR0RHHJpY2hAYW5u
|
||||||
|
ZXhpYS5vcmcACgkQkXOPc+G3aKBAzw//ZRKoT32ixiD1aL21Trbv8NTobPFy9We1
|
||||||
|
hjdt6CYETBC71aTiaDONhBbXzyX0gv31WmCHWLYrxaE9edS9oaUD4l763ZCuYr9O
|
||||||
|
6JR3CW8NG9/soh5INESzl7cm0i4WHofhKjMkmoZt+vPlCnBfUuZ+GOQHlzYNc41N
|
||||||
|
3h2rjuIZZNRD0op6FmgnrY0Y2IYYtzR1kiUh47JQanHwwDHhdaaz4x348LQk/FlI
|
||||||
|
s0qQv8wYe2kRfLvCgNVaPgywp8dx3cb+JmxcRcmolHLWM171XBkrYl2cJjBnYBAr
|
||||||
|
5/pYU6wpzun3R53fTxHRBddLkLOOy2mbGDvyc0lsnc2Jh0RWKEThvC9A216KcSyL
|
||||||
|
vIVwXnH5zq4mbEApK6G+hO34SWOmT5f8sIWm2vfsaQ7QncBO3fmBkge1o9roGmHy
|
||||||
|
97nOPbx2+070gGz5tvCdCcZ4cq+K4Xh4OjikrBj/O0vmeA6c7+REJulKTxF9Dp4H
|
||||||
|
e+zswAGo4kN7uGMSET3U8nbuWxs+RZfYGqCX1ivkfRBj8271CEh3rFNrlAxq3M8v
|
||||||
|
dEUB8d3LIwlKMrc/ZpjL3x9zuM7dR7ZTQkYKJGRIL5GPUO9jwRDLBPCnx5Krjiyk
|
||||||
|
JyN7Uxd52k09GDcLNKHDa3qLiU7Ye+DFTZqF23DOgXwlEWP/Gbg0Xe6DRVIM2gwM
|
||||||
|
ateF79dey/4=
|
||||||
|
=Le0m
|
||||||
|
-----END PGP SIGNATURE-----
|
Loading…
Reference in new issue