commit 5ce8e9e4fcdc8338d068753cc9d22eca14042317 Author: CentOS Sources Date: Tue Nov 15 01:24:58 2022 -0500 import libnbd-1.12.6-1.el9 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea01122 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +SOURCES/libguestfs.keyring +SOURCES/libnbd-1.12.6.tar.gz diff --git a/.libnbd.metadata b/.libnbd.metadata new file mode 100644 index 0000000..f20759f --- /dev/null +++ b/.libnbd.metadata @@ -0,0 +1,2 @@ +cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring +2d4eb0846d51c25fa7d04295972cbb5a617984ef SOURCES/libnbd-1.12.6.tar.gz diff --git a/SOURCES/0001-Add-nbddump-tool.patch b/SOURCES/0001-Add-nbddump-tool.patch new file mode 100644 index 0000000..5600ba9 --- /dev/null +++ b/SOURCES/0001-Add-nbddump-tool.patch @@ -0,0 +1,1223 @@ +From 90fd39da16256407b9229cd17a830739b03629d6 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 09:07:27 +0100 +Subject: [PATCH] Add nbddump tool + +You can already do this operation using: + + nbdcopy -- $uri - | hexdump -C + +but that is slow, especially for large, sparse disks. This tool uses +sparseness information to skip zero sections of the disk, and it has +nice colourized output. + +(cherry picked from commit c4107b9a40d6451630dcccf1bf6596c8e56420be) +--- + .gitignore | 3 + + Makefile.am | 1 + + README | 2 + + bash-completion/Makefile.am | 9 +- + bash-completion/nbdsh | 6 + + configure.ac | 1 + + copy/nbdcopy.pod | 3 +- + docs/libnbd.pod | 1 + + dump/Makefile.am | 79 ++++++ + dump/dump-data.sh | 57 +++++ + dump/dump-empty-qcow2.sh | 46 ++++ + dump/dump-pattern.sh | 56 +++++ + dump/dump.c | 464 ++++++++++++++++++++++++++++++++++++ + dump/nbddump.pod | 116 +++++++++ + dump/test-long-options.sh | 35 +++ + dump/test-short-options.sh | 35 +++ + dump/test-version.sh | 33 +++ + fuse/nbdfuse.pod | 1 + + info/nbdinfo.pod | 1 + + run.in | 1 + + sh/nbdsh.pod | 1 + + 21 files changed, 947 insertions(+), 4 deletions(-) + create mode 100644 dump/Makefile.am + create mode 100755 dump/dump-data.sh + create mode 100755 dump/dump-empty-qcow2.sh + create mode 100755 dump/dump-pattern.sh + create mode 100644 dump/dump.c + create mode 100644 dump/nbddump.pod + create mode 100755 dump/test-long-options.sh + create mode 100755 dump/test-short-options.sh + create mode 100755 dump/test-version.sh + +diff --git a/.gitignore b/.gitignore +index 498eabc..3771655 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -29,6 +29,7 @@ Makefile.in + /aclocal.m4 + /autom4te.cache + /bash-completion/nbdcopy ++/bash-completion/nbddump + /bash-completion/nbdfuse + /bash-completion/nbdinfo + /common/include/test-array-size +@@ -57,6 +58,8 @@ Makefile.in + !/docs/nbd_close.3 + !/docs/nbd_create.pod + !/docs/nbd_get_err??.3 ++/dump/nbddump ++/dump/nbddump.1 + /examples/aio-connect-read + /examples/batched-read-write + /examples/connect-command +diff --git a/Makefile.am b/Makefile.am +index 303b95c..9e7a281 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -46,6 +46,7 @@ SUBDIRS = \ + sh \ + info \ + copy \ ++ dump \ + fuse \ + ocaml \ + ocaml/examples \ +diff --git a/README b/README +index e50c19d..4f9298e 100644 +--- a/README ++++ b/README +@@ -23,6 +23,8 @@ The key features are: + + * Copying tool (nbdcopy) for high performance copying and streaming. + ++ * Hexdump tool (nbddump) to print NBD content. ++ + * Query tool (nbdinfo) to query NBD servers. + + * FUSE support (nbdfuse) to mount NBD in the local filesystem. +diff --git a/bash-completion/Makefile.am b/bash-completion/Makefile.am +index 41d7b13..cab8ffb 100644 +--- a/bash-completion/Makefile.am ++++ b/bash-completion/Makefile.am +@@ -24,17 +24,20 @@ EXTRA_DIST = \ + + if HAVE_BASH_COMPLETION + +-bashcomp_DATA = nbdfuse nbdsh ++bashcomp_DATA = nbddump nbdfuse nbdsh + + if HAVE_LIBXML2 + bashcomp_DATA += nbdcopy nbdinfo + endif HAVE_LIBXML2 + +- + nbdcopy: nbdsh + rm -f $@ + $(LN_S) $(srcdir)/nbdsh $@ + ++nbddump: nbdsh ++ rm -f $@ ++ $(LN_S) $(srcdir)/nbdsh $@ ++ + nbdfuse: nbdsh + rm -f $@ + $(LN_S) $(srcdir)/nbdsh $@ +@@ -43,6 +46,6 @@ nbdinfo: nbdsh + rm -f $@ + $(LN_S) $(srcdir)/nbdsh $@ + +-CLEANFILES += nbdcopy nbdfuse nbdinfo ++CLEANFILES += nbdcopy nbddump nbdfuse nbdinfo + + endif +diff --git a/bash-completion/nbdsh b/bash-completion/nbdsh +index a740be9..a342003 100644 +--- a/bash-completion/nbdsh ++++ b/bash-completion/nbdsh +@@ -47,6 +47,11 @@ _nbdcopy () + _libnbd_command nbdcopy + } + ++_nbddump () ++{ ++ _libnbd_command nbddump ++} ++ + _nbdfuse () + { + _libnbd_command nbdfuse +@@ -64,6 +69,7 @@ _nbdsh () + + # Install the handler function. + complete -o default -F _nbdcopy nbdcopy ++complete -o default -F _nbddump nbddump + complete -o default -F _nbdfuse nbdfuse + complete -o default -F _nbdinfo nbdinfo + complete -o default -F _nbdsh nbdsh +diff --git a/configure.ac b/configure.ac +index b1bfaac..49ca8ab 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -574,6 +574,7 @@ AC_CONFIG_FILES([Makefile + common/utils/Makefile + copy/Makefile + docs/Makefile ++ dump/Makefile + examples/Makefile + fuse/Makefile + fuzzing/Makefile +diff --git a/copy/nbdcopy.pod b/copy/nbdcopy.pod +index 7fe3fd1..fd10f7c 100644 +--- a/copy/nbdcopy.pod ++++ b/copy/nbdcopy.pod +@@ -285,7 +285,7 @@ Some examples follow. + In this example, L is run as a subprocess. The + subprocess opens F and exposes it as NBD to nbdcopy. + nbdcopy streams this to stdout (C<->) into the pipe which is read by +-L. ++L. (See also L) + + =head2 nbdcopy -- [ qemu-nbd -f qcow2 disk.qcow2 ] [ nbdkit memory 1G ] + +@@ -299,6 +299,7 @@ so this command has no overall effect, but is useful for testing. + =head1 SEE ALSO + + L, ++L, + L, + L, + L, +diff --git a/docs/libnbd.pod b/docs/libnbd.pod +index 13facc6..076cafb 100644 +--- a/docs/libnbd.pod ++++ b/docs/libnbd.pod +@@ -1044,6 +1044,7 @@ L. + + L, + L, ++L, + L, + L, + L, +diff --git a/dump/Makefile.am b/dump/Makefile.am +new file mode 100644 +index 0000000..9fd4fed +--- /dev/null ++++ b/dump/Makefile.am +@@ -0,0 +1,79 @@ ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++include $(top_srcdir)/subdir-rules.mk ++ ++EXTRA_DIST = \ ++ dump-data.sh \ ++ dump-empty-qcow2.sh \ ++ dump-pattern.sh \ ++ nbddump.pod \ ++ test-long-options.sh \ ++ test-short-options.sh \ ++ test-version.sh \ ++ $(NULL) ++ ++bin_PROGRAMS = nbddump ++ ++nbddump_SOURCES = \ ++ dump.c \ ++ $(NULL) ++nbddump_CPPFLAGS = \ ++ -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/include \ ++ -I$(top_srcdir)/common/utils \ ++ $(NULL) ++nbddump_CFLAGS = \ ++ $(WARNINGS_CFLAGS) \ ++ $(NULL) ++nbddump_LDADD = \ ++ $(top_builddir)/common/utils/libutils.la \ ++ $(top_builddir)/lib/libnbd.la \ ++ $(NULL) ++ ++if HAVE_POD ++ ++man_MANS = \ ++ nbddump.1 \ ++ $(NULL) ++ ++nbddump.1: nbddump.pod $(top_builddir)/podwrapper.pl ++ $(PODWRAPPER) --section=1 --man $@ \ ++ --html $(top_builddir)/html/$@.html \ ++ $< ++ ++endif HAVE_POD ++ ++TESTS_ENVIRONMENT = \ ++ LIBNBD_DEBUG=1 \ ++ $(MALLOC_CHECKS) \ ++ EXPECTED_VERSION=$(VERSION) \ ++ QEMU_NBD=$(QEMU_NBD) \ ++ $(NULL) ++LOG_COMPILER = $(top_builddir)/run ++ ++TESTS = \ ++ dump-data.sh \ ++ dump-empty-qcow2.sh \ ++ dump-pattern.sh \ ++ test-long-options.sh \ ++ test-short-options.sh \ ++ test-version.sh \ ++ $(NULL) ++ ++check-valgrind: ++ LIBNBD_VALGRIND=1 $(MAKE) check +diff --git a/dump/dump-data.sh b/dump/dump-data.sh +new file mode 100755 +index 0000000..23d09da +--- /dev/null ++++ b/dump/dump-data.sh +@@ -0,0 +1,57 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires nbdkit --version ++requires nbdkit data --dump-plugin ++ ++output=dump-data.out ++rm -f $output ++cleanup_fn rm -f $output ++ ++nbdkit -U - data data=' ++ @32768 1 ++ @65535 "hello, world!" ++ @17825790 "spanning buffer boundary" ++ @20000000 0 ++' --run 'nbddump "$uri"' > $output ++ ++cat $output ++ ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h| ++0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....| ++0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp| ++0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo| ++0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........| ++0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++0001312d00: 00 |. |' ]; then ++ echo "$0: unexpected output from nbddump command" ++ exit 1 ++fi +diff --git a/dump/dump-empty-qcow2.sh b/dump/dump-empty-qcow2.sh +new file mode 100755 +index 0000000..c9e583b +--- /dev/null ++++ b/dump/dump-empty-qcow2.sh +@@ -0,0 +1,46 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires $QEMU_NBD --version ++requires qemu-img --version ++ ++file=dump-empty-qcow2.qcow2 ++output=dump-empty-qcow2.out ++rm -f $file $output ++cleanup_fn rm -f $file $output ++ ++size=1G ++ ++# Create a large, empty qcow2 file. ++qemu-img create -f qcow2 $file $size ++ ++# Dump it and check the output. ++nbddump -- [ $QEMU_NBD -r -f qcow2 $file ] > $output ++cat $output ++ ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then ++ echo "$0: unexpected output from nbddump command" ++ exit 1 ++fi +diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh +new file mode 100755 +index 0000000..e4016a8 +--- /dev/null ++++ b/dump/dump-pattern.sh +@@ -0,0 +1,56 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires nbdkit --version ++requires nbdkit pattern --dump-plugin ++ ++output=dump-pattern.out ++rm -f $output ++cleanup_fn rm -f $output ++ ++nbdkit -U - pattern size=299 --run 'nbddump "$uri"' > $output ++ ++cat $output ++ ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| ++0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................| ++0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(| ++0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8| ++0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H| ++0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X| ++0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h| ++0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x| ++0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................| ++0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................| ++00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................| ++00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................| ++00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................| ++00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................| ++00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................| ++00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................| ++0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................| ++0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................| ++0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then ++ echo "$0: unexpected output from nbddump command" ++ exit 1 ++fi +diff --git a/dump/dump.c b/dump/dump.c +new file mode 100644 +index 0000000..76af04c +--- /dev/null ++++ b/dump/dump.c +@@ -0,0 +1,464 @@ ++/* NBD client library in userspace ++ * Copyright (C) 2020-2022 Red Hat Inc. ++ * ++ * This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public ++ * License as published by the Free Software Foundation; either ++ * version 2 of the License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ * Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with this library; if not, write to the Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "minmax.h" ++#include "rounding.h" ++#include "version.h" ++#include "vector.h" ++ ++DEFINE_VECTOR_TYPE (uint32_vector, uint32_t) ++ ++static const char *progname; ++static struct nbd_handle *nbd; ++static bool colour; ++static uint64_t limit = UINT64_MAX; /* --length (unlimited by default) */ ++static int64_t size; /* actual size */ ++static bool can_meta_context; /* did we get extent data? */ ++ ++/* See do_connect () */ ++static enum { MODE_URI = 1, MODE_SQUARE_BRACKET } mode; ++static char **args; ++ ++/* Read buffer. */ ++static unsigned char buffer[16*1024*1024]; ++ ++static void do_connect (void); ++static void do_dump (void); ++static void catch_signal (int); ++ ++static void __attribute__((noreturn)) ++usage (FILE *fp, int exitcode) ++{ ++ fprintf (fp, ++"\n" ++"Hexdump the content of a disk over NBD:\n" ++"\n" ++" nbddump NBD-URI | [ CMD ARGS ... ]\n" ++"\n" ++"Other options:\n" ++"\n" ++" nbddump --help\n" ++" nbddump --version\n" ++"\n" ++"Examples:\n" ++"\n" ++" nbddump nbd://localhost\n" ++" nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ]\n" ++"\n" ++"Please read the nbddump(1) manual page for full usage.\n" ++"\n" ++); ++ exit (exitcode); ++} ++ ++int ++main (int argc, char *argv[]) ++{ ++ enum { ++ HELP_OPTION = CHAR_MAX + 1, ++ LONG_OPTIONS, ++ SHORT_OPTIONS, ++ COLOUR_OPTION, ++ NO_COLOUR_OPTION, ++ }; ++ const char *short_options = "n:V"; ++ const struct option long_options[] = { ++ { "help", no_argument, NULL, HELP_OPTION }, ++ { "long-options", no_argument, NULL, LONG_OPTIONS }, ++ { "short-options", no_argument, NULL, SHORT_OPTIONS }, ++ { "version", no_argument, NULL, 'V' }, ++ ++ { "color", no_argument, NULL, COLOUR_OPTION }, ++ { "colors", no_argument, NULL, COLOUR_OPTION }, ++ { "colour", no_argument, NULL, COLOUR_OPTION }, ++ { "colours", no_argument, NULL, COLOUR_OPTION }, ++ { "no-color", no_argument, NULL, NO_COLOUR_OPTION }, ++ { "no-colors", no_argument, NULL, NO_COLOUR_OPTION }, ++ { "no-colour", no_argument, NULL, NO_COLOUR_OPTION }, ++ { "no-colours", no_argument, NULL, NO_COLOUR_OPTION }, ++ { "length", required_argument, NULL, 'n' }, ++ { "limit", required_argument, NULL, 'n' }, ++ { NULL } ++ }; ++ int c; ++ size_t i; ++ ++ progname = argv[0]; ++ colour = isatty (STDOUT_FILENO); ++ ++ for (;;) { ++ c = getopt_long (argc, argv, short_options, long_options, NULL); ++ if (c == -1) ++ break; ++ ++ switch (c) { ++ case HELP_OPTION: ++ usage (stdout, EXIT_SUCCESS); ++ ++ case LONG_OPTIONS: ++ for (i = 0; long_options[i].name != NULL; ++i) { ++ if (strcmp (long_options[i].name, "long-options") != 0 && ++ strcmp (long_options[i].name, "short-options") != 0) ++ printf ("--%s\n", long_options[i].name); ++ } ++ exit (EXIT_SUCCESS); ++ ++ case SHORT_OPTIONS: ++ for (i = 0; short_options[i]; ++i) { ++ if (short_options[i] != ':' && short_options[i] != '+') ++ printf ("-%c\n", short_options[i]); ++ } ++ exit (EXIT_SUCCESS); ++ ++ case COLOUR_OPTION: ++ colour = true; ++ break; ++ ++ case NO_COLOUR_OPTION: ++ colour = false; ++ break; ++ ++ case 'n': ++ /* XXX Allow human sizes here. */ ++ if (sscanf (optarg, "%" SCNu64, &limit) != 1) { ++ fprintf (stderr, "%s: could not parse --length option: %s\n", ++ progname, optarg); ++ exit (EXIT_FAILURE); ++ } ++ break; ++ ++ case 'V': ++ display_version ("nbddump"); ++ exit (EXIT_SUCCESS); ++ ++ default: ++ usage (stderr, EXIT_FAILURE); ++ } ++ } ++ ++ /* Is it a URI or subprocess? */ ++ if (argc - optind >= 3 && ++ strcmp (argv[optind], "[") == 0 && ++ strcmp (argv[argc-1], "]") == 0) { ++ mode = MODE_SQUARE_BRACKET; ++ argv[argc-1] = NULL; ++ args = &argv[optind+1]; ++ } ++ else if (argc - optind == 1) { ++ mode = MODE_URI; ++ args = &argv[optind]; ++ } ++ else { ++ usage (stderr, EXIT_FAILURE); ++ } ++ ++ /* Open the NBD side. */ ++ nbd = nbd_create (); ++ if (nbd == NULL) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ nbd_set_uri_allow_local_file (nbd, true); /* Allow ?tls-psk-file. */ ++ nbd_add_meta_context (nbd, LIBNBD_CONTEXT_BASE_ALLOCATION); ++ ++ /* Connect to the server. */ ++ do_connect (); ++ can_meta_context = ++ nbd_can_meta_context (nbd, LIBNBD_CONTEXT_BASE_ALLOCATION) > 0; ++ ++ /* Get the size. */ ++ size = nbd_get_size (nbd); ++ if (size == -1) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ ++ /* Before dumping, make sure we restore the terminal on ^C etc. */ ++ signal (SIGINT, catch_signal); ++ signal (SIGQUIT, catch_signal); ++ signal (SIGTERM, catch_signal); ++ signal (SIGHUP, catch_signal); ++ ++ /* Dump the content. */ ++ do_dump (); ++ ++ nbd_shutdown (nbd, 0); ++ nbd_close (nbd); ++ ++ exit (EXIT_SUCCESS); ++} ++ ++/* Connect the handle to the server. */ ++static void ++do_connect (void) ++{ ++ int r; ++ ++ switch (mode) { ++ case MODE_URI: /* NBD-URI */ ++ r = nbd_connect_uri (nbd, args[0]); ++ break; ++ ++ case MODE_SQUARE_BRACKET: /* [ CMD ARGS ... ] */ ++ r = nbd_connect_systemd_socket_activation (nbd, args); ++ break; ++ ++ default: ++ abort (); ++ } ++ ++ if (r == -1) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++} ++ ++/* Various ANSI colours, suppressed if --no-colour / not tty output. */ ++static void ++ansi_restore (void) ++{ ++ if (colour) ++ fputs ("\033[0m", stdout); ++} ++ ++static void ++ansi_blue (void) ++{ ++ if (colour) ++ fputs ("\033[1;34m", stdout); ++} ++ ++static void ++ansi_green (void) ++{ ++ if (colour) ++ fputs ("\033[0;32m", stdout); ++} ++ ++static void ++ansi_magenta (void) ++{ ++ if (colour) ++ fputs ("\033[1;35m", stdout); ++} ++ ++static void ++ansi_red (void) ++{ ++ if (colour) ++ fputs ("\033[1;31m", stdout); ++} ++ ++static void ++ansi_grey (void) ++{ ++ if (colour) ++ fputs ("\033[0;90m", stdout); ++} ++ ++static void ++catch_signal (int sig) ++{ ++ printf ("\n"); ++ ansi_restore (); ++ fflush (stdout); ++ _exit (EXIT_FAILURE); ++} ++ ++/* Read the extent map for the next block and return true if it is all ++ * zeroes. This is conservative and returns false if we did not get ++ * the full extent map from the server, or if the server doesn't ++ * support base:allocation at all. ++ */ ++static int ++extent_callback (void *user_data, const char *metacontext, ++ uint64_t offset, ++ uint32_t *entries, size_t nr_entries, ++ int *error) ++{ ++ uint32_vector *list = user_data; ++ size_t i; ++ ++ if (strcmp (metacontext, LIBNBD_CONTEXT_BASE_ALLOCATION) != 0) ++ return 0; ++ ++ /* Just append the entries we got to the list. */ ++ for (i = 0; i < nr_entries; ++i) { ++ if (uint32_vector_append (list, entries[i]) == -1) { ++ perror ("realloc"); ++ exit (EXIT_FAILURE); ++ } ++ } ++ return 0; ++} ++ ++static bool ++test_all_zeroes (uint64_t offset, size_t count) ++{ ++ uint32_vector entries = empty_vector; ++ size_t i; ++ uint64_t count_read; ++ ++ if (!can_meta_context) ++ return false; ++ ++ /* Get the extent map for the block. Note the server doesn't need ++ * to return all requested data here. If it does not then we return ++ * false, causing the main code to do a full read. We could be ++ * smarter and keep asking the server (XXX). ++ */ ++ if (nbd_block_status (nbd, count, offset, ++ (nbd_extent_callback) { ++ .callback = extent_callback, ++ .user_data = &entries }, ++ 0) == -1) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ ++ count_read = 0; ++ for (i = 0; i < entries.len; i += 2) { ++ uint32_t len = entries.ptr[i]; ++ uint32_t type = entries.ptr[i+1]; ++ ++ count_read += len; ++ if (!(type & 2)) /* not zero */ ++ return false; ++ } ++ ++ /* Did we read at least the whole range wanted? */ ++ if (count_read < count) ++ return false; ++ ++ /* If we got here, we read the whole range and it was all zeroes. */ ++ return true; ++} ++ ++/* Hexdump the NBD data. ++ * ++ * XXX In future we could do this all asynch (including writing to ++ * stdout) which could make it very efficient. ++ */ ++static void ++do_dump (void) ++{ ++ /* If --no-colour, don't use unicode in the output. */ ++ const char *splat = colour ? "☆" : "*"; ++ const char *pipe = colour ? "│" : "|"; ++ const char *dot = colour ? "·" : "."; ++ uint64_t offset = 0; ++ uint64_t count = size > limit ? limit : size; ++ size_t i, j, n; ++ char last[16]; ++ bool printed_splat = false, same; ++ ++ while (count) { ++ n = MIN (count, sizeof buffer); ++ ++ if (! test_all_zeroes (offset, n)) { ++ if (nbd_pread (nbd, buffer, n, offset, 0) == -1) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ } ++ else { ++ memset (buffer, 0, n); ++ } ++ ++ /* Make sure a multiple of 16 bytes gets written to the buffer. */ ++ if (n & 15) ++ memset (&buffer[n], 0, 16 - (n & 15)); ++ ++ for (i = 0; i < n; i += 16) { ++ /* Is this line the same as the last line? (Squashing) */ ++ same = ++ offset + i > 0 && /* first line is never squashed */ ++ offset + i + 16 < size && /* last line is never squashed */ ++ memcmp (&buffer[i], last, 16) == 0; ++ if (same) { ++ if (!printed_splat) { ++ printf ("%s\n", splat); ++ printed_splat = true; ++ } ++ continue; ++ } ++ printed_splat = false; ++ memcpy (last, &buffer[i], 16); /* Save the current line. */ ++ ++ /* Print the offset. */ ++ ansi_green (); ++ printf ("%010zx", offset + i); ++ ansi_grey (); ++ printf (": "); ++ ++ /* Print the hex codes. */ ++ for (j = i; j < MIN (i+16, n); ++j) { ++ if (buffer[j]) ++ ansi_blue (); ++ else ++ ansi_grey (); ++ printf ("%02x ", buffer[j]); ++ } ++ ansi_grey (); ++ for (; j < i+16; ++j) ++ printf (" "); ++ ++ /* Print the ASCII codes. */ ++ printf ("%s", pipe); ++ for (j = i; j < MIN (i+16, n); ++j) { ++ char c = (char) buffer[j]; ++ if (isalnum (c)) { ++ ansi_red (); ++ printf ("%c", c); ++ } ++ else if (isprint (c)) { ++ ansi_magenta (); ++ printf ("%c", c); ++ } ++ else { ++ ansi_grey (); ++ printf ("%s", dot); ++ } ++ } ++ ansi_grey (); ++ for (; j < i+16; ++j) ++ printf (" "); ++ printf ("%s\n", pipe); ++ ansi_restore (); ++ } ++ ++ offset += n; ++ count -= n; ++ } ++} +diff --git a/dump/nbddump.pod b/dump/nbddump.pod +new file mode 100644 +index 0000000..5d7864d +--- /dev/null ++++ b/dump/nbddump.pod +@@ -0,0 +1,116 @@ ++=head1 NAME ++ ++nbddump - hexdump the content of a disk over NBD ++ ++=head1 SYNOPSIS ++ ++ nbddump NBD ++ ++C is an NBD URI or subprocess: ++ ++ NBD := nbd://... | nbd+unix:// (or other URI formats) ++ | [ CMD ARGS ... ] ++ ++=for paragraph ++ ++ nbddump --help ++ ++=for paragraph ++ ++ nbddump --version ++ ++=head1 DESCRIPTION ++ ++nbddump prints the content of a disk from an NBD server using the ++usual hexdump format: ++ ++ $ nbddump nbd://localhost ++ 0000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │················│ ++ 0010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │················│ ++ ☆ ++ 0100: 68 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 │hello, world!···│ ++ 0110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │················│ ++ ☆ ++ 1000: 00 00 00 21 │···! │ ++ ++=head2 Output format ++ ++The first field (before the C<:>) is the offset within the file, in ++hexadecimal. ++ ++The second field shows the hex codes of bytes read from the file. ++ ++The third field shows the ASCII equivalent characters (if printable). ++ ++A splat character (C<☆>) indicates lines of repeated output which have ++been squashed. (Note this is not just for lines of zero bytes, but ++any case where the next line shown would be the same as the previous ++line.) ++ ++=head2 Subprocess ++ ++nbddump can also run an NBD server as a subprocess. This requires an ++NBD server which understands systemd socket activation, such as ++L or L. ++ ++For example, to dump out a qcow2 file as raw data: ++ ++ nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ] ++ ++Note that S> are separate parameters, and must be ++surrounded by spaces. C<--> separates nbddump parameters from ++subprocess parameters. ++ ++=head1 OPTIONS ++ ++=over 4 ++ ++=item B<--help> ++ ++Display brief command line help and exit. ++ ++=item B<--color> ++ ++=item B<--colour> ++ ++=item B<--no-color> ++ ++=item B<--no-colour> ++ ++Enable or disable ANSI colours in output. By default we use colours ++if the output seems to be a terminal, and disable them if not. ++ ++=item B<--length=>N ++ ++=item B<-n> N ++ ++Dump up to I bytes and then stop. ++ ++=item B<-V> ++ ++=item B<--version> ++ ++Display the package name and version and exit. ++ ++=back ++ ++=head1 SEE ALSO ++ ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L. ++ ++=head1 AUTHORS ++ ++Richard W.M. Jones ++ ++=head1 COPYRIGHT ++ ++Copyright (C) 2022 Red Hat Inc. +diff --git a/dump/test-long-options.sh b/dump/test-long-options.sh +new file mode 100755 +index 0000000..924c8f5 +--- /dev/null ++++ b/dump/test-long-options.sh +@@ -0,0 +1,35 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2019-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++# Test that nbddump --long-options looks sane. ++ ++. ../tests/functions.sh ++set -e ++set -x ++ ++output=test-long-options.out ++cleanup_fn rm -f $output ++ ++$VG nbddump --long-options > $output ++if [ $? != 0 ]; then ++ echo "$0: unexpected exit status" ++ fail=1 ++fi ++cat $output ++grep -- --length $output ++grep -- --version $output +diff --git a/dump/test-short-options.sh b/dump/test-short-options.sh +new file mode 100755 +index 0000000..325f7df +--- /dev/null ++++ b/dump/test-short-options.sh +@@ -0,0 +1,35 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2019-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++# Test that nbddump --short-options looks sane. ++ ++. ../tests/functions.sh ++set -e ++set -x ++ ++output=test-short-options.out ++cleanup_fn rm -f $output ++ ++$VG nbddump --short-options > $output ++if [ $? != 0 ]; then ++ echo "$0: unexpected exit status" ++ fail=1 ++fi ++cat $output ++grep -- -n $output ++grep -- -V $output +diff --git a/dump/test-version.sh b/dump/test-version.sh +new file mode 100755 +index 0000000..fce4ed1 +--- /dev/null ++++ b/dump/test-version.sh +@@ -0,0 +1,33 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2019 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++# Test that nbddump --version looks sane. ++ ++fail=0 ++output=$($VG nbddump --version) ++if [ $? != 0 ]; then ++ echo "$0: unexpected exit status" ++ fail=1 ++fi ++if [ "$output" != "nbddump $EXPECTED_VERSION ++libnbd $EXPECTED_VERSION" ]; then ++ echo "$0: unexpected output" ++ fail=1 ++fi ++echo "$output" ++exit $fail +diff --git a/fuse/nbdfuse.pod b/fuse/nbdfuse.pod +index 7c1c817..daa79c1 100644 +--- a/fuse/nbdfuse.pod ++++ b/fuse/nbdfuse.pod +@@ -412,6 +412,7 @@ The differences from nbdfuse are similar to the list above. + + L, + L, ++L, + L, + L, + L, +diff --git a/info/nbdinfo.pod b/info/nbdinfo.pod +index 649cf1c..4733ecd 100644 +--- a/info/nbdinfo.pod ++++ b/info/nbdinfo.pod +@@ -407,6 +407,7 @@ Display the package name and version and exit. + + L, + L, ++L, + L, + L, + L, +diff --git a/run.in b/run.in +index 8a21906..2a171e5 100755 +--- a/run.in ++++ b/run.in +@@ -58,6 +58,7 @@ b="$(cd @abs_builddir@ && pwd)" + + # Set the PATH to contain all libnbd binaries. + prepend PATH "$b/copy" ++prepend PATH "$b/dump" + prepend PATH "$b/fuse" + prepend PATH "$b/info" + prepend PATH "$b/sh" +diff --git a/sh/nbdsh.pod b/sh/nbdsh.pod +index ca5d6af..c9dac4a 100644 +--- a/sh/nbdsh.pod ++++ b/sh/nbdsh.pod +@@ -147,6 +147,7 @@ L. + L, + L, + L, ++L, + L, + L, + L. +-- +2.31.1 + diff --git a/SOURCES/0002-dump-Visually-separate-columns-0-7-and-8-15.patch b/SOURCES/0002-dump-Visually-separate-columns-0-7-and-8-15.patch new file mode 100644 index 0000000..67fc1af --- /dev/null +++ b/SOURCES/0002-dump-Visually-separate-columns-0-7-and-8-15.patch @@ -0,0 +1,153 @@ +From ec947323528725fcf12b5b9ba32b02d36dbd9621 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 21:09:39 +0100 +Subject: [PATCH] dump: Visually separate columns 0-7 and 8-15 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Before: + +0000090000: 68 65 72 65 20 77 65 20 61 72 65 00 68 65 72 65 │... +0000090010: 20 77 65 20 61 72 65 00 68 65 72 65 20 77 65 20 │... +0000090020: 61 72 65 00 68 65 72 65 20 77 65 20 61 72 65 00 │... + +After: + +0000090000: 68 65 72 65 20 77 65 20 61 72 65 00 68 65 72 65 │... +0000090010: 20 77 65 20 61 72 65 00 68 65 72 65 20 77 65 20 │... +0000090020: 61 72 65 00 68 65 72 65 20 77 65 20 61 72 65 00 │... + +Updates: commit c4107b9a40d6451630dcccf1bf6596c8e56420be +(cherry picked from commit 315a637d3eae003c1d84eb1b88a7b47b534f1e80) +--- + dump/dump-data.sh | 22 +++++++++++----------- + dump/dump-empty-qcow2.sh | 4 ++-- + dump/dump-pattern.sh | 38 +++++++++++++++++++------------------- + dump/dump.c | 5 ++++- + 4 files changed, 36 insertions(+), 33 deletions(-) + +diff --git a/dump/dump-data.sh b/dump/dump-data.sh +index 23d09da..955cd3b 100755 +--- a/dump/dump-data.sh ++++ b/dump/dump-data.sh +@@ -37,21 +37,21 @@ nbdkit -U - data data=' + + cat $output + +-if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +-0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h| +-0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....| +-0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h| ++0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....| ++0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp| +-0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo| +-0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........| +-0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp| ++0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo| ++0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........| ++0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-0001312d00: 00 |. |' ]; then ++0001312d00: 00 |. |' ]; then + echo "$0: unexpected output from nbddump command" + exit 1 + fi +diff --git a/dump/dump-empty-qcow2.sh b/dump/dump-empty-qcow2.sh +index c9e583b..472b6eb 100755 +--- a/dump/dump-empty-qcow2.sh ++++ b/dump/dump-empty-qcow2.sh +@@ -38,9 +38,9 @@ qemu-img create -f qcow2 $file $size + nbddump -- [ $QEMU_NBD -r -f qcow2 $file ] > $output + cat $output + +-if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then ++003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then + echo "$0: unexpected output from nbddump command" + exit 1 + fi +diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh +index e4016a8..d512b77 100755 +--- a/dump/dump-pattern.sh ++++ b/dump/dump-pattern.sh +@@ -32,25 +32,25 @@ nbdkit -U - pattern size=299 --run 'nbddump "$uri"' > $output + + cat $output + +-if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| +-0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................| +-0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(| +-0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8| +-0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H| +-0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X| +-0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h| +-0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x| +-0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................| +-0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................| +-00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................| +-00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................| +-00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................| +-00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................| +-00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................| +-00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................| +-0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................| +-0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................| +-0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| ++0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................| ++0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(| ++0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8| ++0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H| ++0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X| ++0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h| ++0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x| ++0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................| ++0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................| ++00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................| ++00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................| ++00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................| ++00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................| ++00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................| ++00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................| ++0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................| ++0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................| ++0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then + echo "$0: unexpected output from nbddump command" + exit 1 + fi +diff --git a/dump/dump.c b/dump/dump.c +index 76af04c..7818f1f 100644 +--- a/dump/dump.c ++++ b/dump/dump.c +@@ -429,10 +429,13 @@ do_dump (void) + else + ansi_grey (); + printf ("%02x ", buffer[j]); ++ if ((j - i) == 7) printf (" "); + } + ansi_grey (); +- for (; j < i+16; ++j) ++ for (; j < i+16; ++j) { + printf (" "); ++ if ((j - i) == 7) printf (" "); ++ } + + /* Print the ASCII codes. */ + printf ("%s", pipe); +-- +2.31.1 + diff --git a/SOURCES/0003-dump-Fix-build-on-i686.patch b/SOURCES/0003-dump-Fix-build-on-i686.patch new file mode 100644 index 0000000..7ec277e --- /dev/null +++ b/SOURCES/0003-dump-Fix-build-on-i686.patch @@ -0,0 +1,38 @@ +From 590e3a010d2c840314702883e44ec9841e3383c6 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 22:27:43 +0100 +Subject: [PATCH] dump: Fix build on i686 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Because we used the wrong printf format, the build would fail on +32 bit architectures but succeed on 64 bit: + +dump.c: In function ‘do_dump’: +dump.c:421:21: error: format ‘%zx’ expects argument of type ‘size_t’, but argument 2 has type ‘uint64_t’ {aka ‘long long unsigned int’} [-Werror=format=] + printf ("%010zx", offset + i); + ~~~~~^ ~~~~~~~~~~ + %010llx + +(cherry picked from commit ce004c329c7fcd6c60d11673b7a5c5ce3414413b) +--- + dump/dump.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/dump/dump.c b/dump/dump.c +index 7818f1f..8bf62f9 100644 +--- a/dump/dump.c ++++ b/dump/dump.c +@@ -418,7 +418,7 @@ do_dump (void) + + /* Print the offset. */ + ansi_green (); +- printf ("%010zx", offset + i); ++ printf ("%010" PRIx64, offset + i); + ansi_grey (); + printf (": "); + +-- +2.31.1 + diff --git a/SOURCES/0004-dump-Fix-tests-on-Debian-10.patch b/SOURCES/0004-dump-Fix-tests-on-Debian-10.patch new file mode 100644 index 0000000..cdb908a --- /dev/null +++ b/SOURCES/0004-dump-Fix-tests-on-Debian-10.patch @@ -0,0 +1,41 @@ +From e7a2815412891d5c13b5b5f0e9aa61882880c87f Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 22:31:00 +0100 +Subject: [PATCH] dump: Fix tests on Debian 10 + +The version of nbdkit on Debian 10 does not set $uri. Check for this +or skip the test. + +(cherry picked from commit 083b1ca30fb5e6e0dc0e4b0eea9ebe8474d3f864) +--- + dump/dump-data.sh | 1 + + dump/dump-pattern.sh | 1 + + 2 files changed, 2 insertions(+) + +diff --git a/dump/dump-data.sh b/dump/dump-data.sh +index 955cd3b..46e4d1e 100755 +--- a/dump/dump-data.sh ++++ b/dump/dump-data.sh +@@ -23,6 +23,7 @@ set -x + + requires nbdkit --version + requires nbdkit data --dump-plugin ++requires nbdkit -U - null --run 'test "$uri" != ""' + + output=dump-data.out + rm -f $output +diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh +index d512b77..e2188ac 100755 +--- a/dump/dump-pattern.sh ++++ b/dump/dump-pattern.sh +@@ -23,6 +23,7 @@ set -x + + requires nbdkit --version + requires nbdkit pattern --dump-plugin ++requires nbdkit -U - null --run 'test "$uri" != ""' + + output=dump-pattern.out + rm -f $output +-- +2.31.1 + diff --git a/SOURCES/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch b/SOURCES/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch new file mode 100644 index 0000000..d868281 --- /dev/null +++ b/SOURCES/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch @@ -0,0 +1,31 @@ +From 7c669783b1b3fab902ce34d7914b62617ed8b263 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 22:35:05 +0100 +Subject: [PATCH] dump/dump-data.sh: Test requires nbdkit 1.22 + +Ubuntu 20.04 has nbdkit 1.16 which lacks support for strings. These +were added in nbdkit 1.22. + +(cherry picked from commit a8fa05ffb8b85f41276ffb52498e4528c08e5f21) +--- + dump/dump-data.sh | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/dump/dump-data.sh b/dump/dump-data.sh +index 46e4d1e..11145b0 100755 +--- a/dump/dump-data.sh ++++ b/dump/dump-data.sh +@@ -25,6 +25,10 @@ requires nbdkit --version + requires nbdkit data --dump-plugin + requires nbdkit -U - null --run 'test "$uri" != ""' + ++# This test requires nbdkit >= 1.22. ++minor=$( nbdkit --dump-config | grep ^version_minor | cut -d= -f2 ) ++requires test $minor -ge 22 ++ + output=dump-data.out + rm -f $output + cleanup_fn rm -f $output +-- +2.31.1 + diff --git a/SOURCES/0006-copy-Store-the-preferred-block-size-in-the-operation.patch b/SOURCES/0006-copy-Store-the-preferred-block-size-in-the-operation.patch new file mode 100644 index 0000000..893a026 --- /dev/null +++ b/SOURCES/0006-copy-Store-the-preferred-block-size-in-the-operation.patch @@ -0,0 +1,163 @@ +From 8dce43a3ea7a529bc37cbe5607a8d52186cc8169 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 28 Jun 2022 18:27:58 +0100 +Subject: [PATCH] copy: Store the preferred block size in the operations struct + +This will be used in a subsequent commit. At the moment the preferred +block size for all sources / destinations is simply calculated and +stored. + +(cherry picked from commit e6c42f8b2d447bbcc659d6dd33be67335834b2e5) +--- + copy/file-ops.c | 4 +++- + copy/main.c | 29 +++++++++++++++++++++++------ + copy/nbd-ops.c | 10 ++++++++++ + copy/nbdcopy.h | 4 +++- + copy/null-ops.c | 1 + + copy/pipe-ops.c | 1 + + 6 files changed, 41 insertions(+), 8 deletions(-) + +diff --git a/copy/file-ops.c b/copy/file-ops.c +index ab37875..34f08e5 100644 +--- a/copy/file-ops.c ++++ b/copy/file-ops.c +@@ -241,13 +241,15 @@ seek_hole_supported (int fd) + + struct rw * + file_create (const char *name, int fd, +- off_t st_size, bool is_block, direction d) ++ off_t st_size, uint64_t preferred, ++ bool is_block, direction d) + { + struct rw_file *rwf = calloc (1, sizeof *rwf); + if (rwf == NULL) { perror ("calloc"); exit (EXIT_FAILURE); } + + rwf->rw.ops = &file_ops; + rwf->rw.name = name; ++ rwf->rw.preferred = preferred; + rwf->fd = fd; + rwf->is_block = is_block; + +diff --git a/copy/main.c b/copy/main.c +index cc379e9..19ec384 100644 +--- a/copy/main.c ++++ b/copy/main.c +@@ -512,10 +512,26 @@ open_local (const char *filename, direction d) + fprintf (stderr, "%s: %s: %m\n", prog, filename); + exit (EXIT_FAILURE); + } +- if (S_ISBLK (stat.st_mode) || S_ISREG (stat.st_mode)) +- return file_create (filename, fd, stat.st_size, S_ISBLK (stat.st_mode), d); +- else { +- /* Probably stdin/stdout, a pipe or a socket. */ ++ if (S_ISREG (stat.st_mode)) /* Regular file. */ ++ return file_create (filename, fd, ++ stat.st_size, (uint64_t) stat.st_blksize, false, d); ++ else if (S_ISBLK (stat.st_mode)) { /* Block device. */ ++ unsigned int blkioopt; ++ ++#ifdef BLKIOOPT ++ if (ioctl (fd, BLKIOOPT, &blkioopt) == -1) { ++ fprintf (stderr, "warning: cannot get optimal I/O size: %s: %m", ++ filename); ++ blkioopt = 4096; ++ } ++#else ++ blkioopt = 4096; ++#endif ++ ++ return file_create (filename, fd, ++ stat.st_size, (uint64_t) blkioopt, true, d); ++ } ++ else { /* Probably stdin/stdout, a pipe or a socket. */ + synchronous = true; /* Force synchronous mode for pipes. */ + return pipe_create (filename, fd); + } +@@ -528,8 +544,9 @@ print_rw (struct rw *rw, const char *prefix, FILE *fp) + char buf[HUMAN_SIZE_LONGEST]; + + fprintf (fp, "%s: %s \"%s\"\n", prefix, rw->ops->ops_name, rw->name); +- fprintf (fp, "%s: size=%" PRIi64 " (%s)\n", +- prefix, rw->size, human_size (buf, rw->size, NULL)); ++ fprintf (fp, "%s: size=%" PRIi64 " (%s), preferred block size=%" PRIu64 "\n", ++ prefix, rw->size, human_size (buf, rw->size, NULL), ++ rw->preferred); + } + + /* Default implementation of rw->ops->get_extents for backends which +diff --git a/copy/nbd-ops.c b/copy/nbd-ops.c +index 3bc26ba..0988634 100644 +--- a/copy/nbd-ops.c ++++ b/copy/nbd-ops.c +@@ -112,12 +112,22 @@ open_one_nbd_handle (struct rw_nbd *rwn) + * the same way. + */ + if (rwn->handles.len == 0) { ++ int64_t block_size; ++ + rwn->can_zero = nbd_can_zero (nbd) > 0; ++ + rwn->rw.size = nbd_get_size (nbd); + if (rwn->rw.size == -1) { + fprintf (stderr, "%s: %s: %s\n", prog, rwn->rw.name, nbd_get_error ()); + exit (EXIT_FAILURE); + } ++ ++ block_size = nbd_get_block_size (nbd, LIBNBD_SIZE_PREFERRED); ++ if (block_size == -1) { ++ fprintf (stderr, "%s: %s: %s\n", prog, rwn->rw.name, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ rwn->rw.preferred = block_size == 0 ? 4096 : block_size; + } + + if (handles_append (&rwn->handles, nbd) == -1) { +diff --git a/copy/nbdcopy.h b/copy/nbdcopy.h +index 19797df..9438cce 100644 +--- a/copy/nbdcopy.h ++++ b/copy/nbdcopy.h +@@ -43,6 +43,7 @@ struct rw { + struct rw_ops *ops; /* Operations. */ + const char *name; /* Printable name, for error messages etc. */ + int64_t size; /* May be -1 for streams. */ ++ uint64_t preferred; /* Preferred block size. */ + /* Followed by private data for the particular subtype. */ + }; + +@@ -53,7 +54,8 @@ typedef enum { READING, WRITING } direction; + + /* Create subtypes. */ + extern struct rw *file_create (const char *name, int fd, +- off_t st_size, bool is_block, direction d); ++ off_t st_size, uint64_t preferred, ++ bool is_block, direction d); + extern struct rw *nbd_rw_create_uri (const char *name, + const char *uri, direction d); + extern struct rw *nbd_rw_create_subprocess (const char **argv, size_t argc, +diff --git a/copy/null-ops.c b/copy/null-ops.c +index 1218a62..99cc9a7 100644 +--- a/copy/null-ops.c ++++ b/copy/null-ops.c +@@ -45,6 +45,7 @@ null_create (const char *name) + rw->rw.ops = &null_ops; + rw->rw.name = name; + rw->rw.size = INT64_MAX; ++ rw->rw.preferred = 4096; + return &rw->rw; + } + +diff --git a/copy/pipe-ops.c b/copy/pipe-ops.c +index 3c8b6c2..3815f82 100644 +--- a/copy/pipe-ops.c ++++ b/copy/pipe-ops.c +@@ -43,6 +43,7 @@ pipe_create (const char *name, int fd) + rwp->rw.ops = &pipe_ops; + rwp->rw.name = name; + rwp->rw.size = -1; ++ rwp->rw.preferred = 4096; + rwp->fd = fd; + return &rwp->rw; + } +-- +2.31.1 + diff --git a/SOURCES/0007-copy-Use-preferred-block-size-for-copying.patch b/SOURCES/0007-copy-Use-preferred-block-size-for-copying.patch new file mode 100644 index 0000000..577f8f1 --- /dev/null +++ b/SOURCES/0007-copy-Use-preferred-block-size-for-copying.patch @@ -0,0 +1,457 @@ +From c8626acc63c4ae1c6cf5d1505e0209ac10f44e81 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 28 Jun 2022 21:58:55 +0100 +Subject: [PATCH] copy: Use preferred block size for copying + +You're not supposed to read or write NBD servers at a granularity less +than the advertised minimum block size. nbdcopy has ignored this +requirement, and this is usually fine because the NBD servers we care +about support 512-byte sector granularity, and never advertise sizes / +extents less granular than sectors (even if it's a bit suboptimal in a +few cases). + +However there is one new case where we do care: When writing to a +compressed qcow2 file, qemu advertises a minimum and preferred block +size of 64K, and it really means it. You cannot write blocks smaller +than this because of the way qcow2 compression is implemented. + +This commit attempts to do the least work possible to fix this. + +The previous multi-thread-copying loop was driven by the extent map +received from the source. I have modified the loop so that it +iterates over request_size blocks. request_size is set from the +command line (--request-size) but will be adjusted upwards if either +the source or destination preferred block size is larger. So this +will always copy blocks which are at least the preferred block size +(except for the very last block of the disk). + +While copying these blocks we consult the source extent map. If it +contains only zero regions covering the whole block (only_zeroes +function) then we can skip straight to zeroing the target +(fill_dst_range_with_zeroes), else we do read + write as before. + +I only modified the multi-thread-copying loop, not the synchronous +loop. That should be updated in the same way later. + +One side effect of this change is it always makes larger requests, +even for regions we know are sparse. This is clear in the +copy-sparse.sh and copy-sparse-allocated.sh tests which were +previously driven by the 32K sparse map granularity of the source. +Without changing these tests, they would make make 256K reads & writes +(and also read from areas of the disk even though we know they are +sparse). I adjusted these tests to use --request-size=32768 to force +the existing behaviour. + +Note this doesn't attempt to limit the maximum block size when reading +or writing. That is for future work. + +This is a partial fix for https://bugzilla.redhat.com/2047660. +Further changes will be required in virt-v2v. + +Link: https://lists.gnu.org/archive/html/qemu-block/2022-01/threads.html#00729 +Link: https://bugzilla.redhat.com/show_bug.cgi?id=2047660 +(cherry picked from commit 4058fe1ff03fb41156b67302ba1006b9d06b0218) +--- + TODO | 4 +- + copy/Makefile.am | 6 +- + copy/copy-file-to-qcow2-compressed.sh | 64 +++++++++++ + copy/copy-sparse-allocated.sh | 4 +- + copy/copy-sparse.sh | 7 +- + copy/main.c | 13 +++ + copy/multi-thread-copying.c | 149 +++++++++++++++++++------- + copy/nbdcopy.pod | 5 +- + 8 files changed, 202 insertions(+), 50 deletions(-) + create mode 100755 copy/copy-file-to-qcow2-compressed.sh + +diff --git a/TODO b/TODO +index 7c9c15e..bc38d70 100644 +--- a/TODO ++++ b/TODO +@@ -28,7 +28,9 @@ Performance: Chart it over various buffer sizes and threads, as that + Examine other fuzzers: https://gitlab.com/akihe/radamsa + + nbdcopy: +- - Minimum/preferred/maximum block size. ++ - Enforce maximum block size. ++ - Synchronous loop should be adjusted to take into account ++ the NBD preferred block size, as was done for multi-thread loop. + - Benchmark. + - Better page cache usage, see nbdkit-file-plugin options + fadvise=sequential cache=none. +diff --git a/copy/Makefile.am b/copy/Makefile.am +index e729f86..25f75c5 100644 +--- a/copy/Makefile.am ++++ b/copy/Makefile.am +@@ -23,6 +23,7 @@ EXTRA_DIST = \ + copy-file-to-nbd.sh \ + copy-file-to-null.sh \ + copy-file-to-qcow2.sh \ ++ copy-file-to-qcow2-compressed.sh \ + copy-nbd-to-block.sh \ + copy-nbd-to-file.sh \ + copy-nbd-to-hexdump.sh \ +@@ -142,7 +143,10 @@ TESTS += \ + $(NULL) + + if HAVE_QEMU_NBD +-TESTS += copy-file-to-qcow2.sh ++TESTS += \ ++ copy-file-to-qcow2.sh \ ++ copy-file-to-qcow2-compressed.sh \ ++ $(NULL) + endif + + if HAVE_GNUTLS +diff --git a/copy/copy-file-to-qcow2-compressed.sh b/copy/copy-file-to-qcow2-compressed.sh +new file mode 100755 +index 0000000..dfe4fa5 +--- /dev/null ++++ b/copy/copy-file-to-qcow2-compressed.sh +@@ -0,0 +1,64 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires $QEMU_NBD --version ++requires nbdkit --exit-with-parent --version ++requires nbdkit sparse-random --dump-plugin ++requires qemu-img --version ++requires stat --version ++ ++file1=copy-file-to-qcow2-compressed.file1 ++file2=copy-file-to-qcow2-compressed.file2 ++rm -f $file1 $file2 ++cleanup_fn rm -f $file1 $file2 ++ ++size=1G ++seed=$RANDOM ++ ++# Create a compressed qcow2 file1. ++# ++# sparse-random files should compress easily because by default each ++# block uses repeated bytes. ++qemu-img create -f qcow2 $file1 $size ++nbdcopy -- [ nbdkit --exit-with-parent sparse-random $size seed=$seed ] \ ++ [ $QEMU_NBD --image-opts driver=compress,file.driver=qcow2,file.file.driver=file,file.file.filename=$file1 ] ++ ++ls -l $file1 ++ ++# Create an uncompressed qcow2 file2 with the same data. ++qemu-img create -f qcow2 $file2 $size ++nbdcopy -- [ nbdkit --exit-with-parent sparse-random $size seed=$seed ] \ ++ [ $QEMU_NBD --image-opts driver=qcow2,file.driver=file,file.filename=$file2 ] ++ ++ls -l $file2 ++ ++# file1 < file2 (shows the compression is having some effect). ++size1="$( stat -c %s $file1 )" ++size2="$( stat -c %s $file2 )" ++if [ $size1 -ge $size2 ]; then ++ echo "$0: qcow2 compression did not make the file smaller" ++ exit 1 ++fi ++ ++# Logical content of the files should be identical. ++qemu-img compare -f qcow2 $file1 -F qcow2 $file2 +diff --git a/copy/copy-sparse-allocated.sh b/copy/copy-sparse-allocated.sh +index 203c3b9..465e347 100755 +--- a/copy/copy-sparse-allocated.sh ++++ b/copy/copy-sparse-allocated.sh +@@ -17,8 +17,6 @@ + # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + # Adapted from copy-sparse.sh. +-# +-# This test depends on the nbdkit default sparse block size (32K). + + . ../tests/functions.sh + +@@ -33,7 +31,7 @@ requires nbdkit eval --version + out=copy-sparse-allocated.out + cleanup_fn rm -f $out + +-$VG nbdcopy --allocated -- \ ++$VG nbdcopy --allocated --request-size=32768 -- \ + [ nbdkit --exit-with-parent data data=' + 1 + @1073741823 1 +diff --git a/copy/copy-sparse.sh b/copy/copy-sparse.sh +index 1a6da86..7912a21 100755 +--- a/copy/copy-sparse.sh ++++ b/copy/copy-sparse.sh +@@ -16,8 +16,6 @@ + # License along with this library; if not, write to the Free Software + # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +-# This test depends on the nbdkit default sparse block size (32K). +- + . ../tests/functions.sh + + set -e +@@ -34,8 +32,9 @@ cleanup_fn rm -f $out + # Copy from a sparse data disk to an nbdkit-eval-plugin instance which + # is logging everything. This allows us to see exactly what nbdcopy + # is writing, to ensure it is writing and zeroing the target as +-# expected. +-$VG nbdcopy -S 0 -- \ ++# expected. Force request size to match nbdkit default sparse ++# allocator block size (32K). ++$VG nbdcopy -S 0 --request-size=32768 -- \ + [ nbdkit --exit-with-parent data data=' + 1 + @1073741823 1 +diff --git a/copy/main.c b/copy/main.c +index 19ec384..0e27db8 100644 +--- a/copy/main.c ++++ b/copy/main.c +@@ -40,6 +40,7 @@ + + #include "ispowerof2.h" + #include "human-size.h" ++#include "minmax.h" + #include "version.h" + #include "nbdcopy.h" + +@@ -379,10 +380,22 @@ main (int argc, char *argv[]) + if (threads < connections) + connections = threads; + ++ /* request_size must always be at least as large as the preferred ++ * size of source & destination. ++ */ ++ request_size = MAX (request_size, src->preferred); ++ request_size = MAX (request_size, dst->preferred); ++ + /* Adapt queue to size to request size if needed. */ + if (request_size > queue_size) + queue_size = request_size; + ++ /* Sparse size (if using) must not be smaller than the destination ++ * preferred size, otherwise we end up creating too small requests. ++ */ ++ if (sparse_size > 0 && sparse_size < dst->preferred) ++ sparse_size = dst->preferred; ++ + /* Truncate the destination to the same size as the source. Only + * has an effect on regular files. + */ +diff --git a/copy/multi-thread-copying.c b/copy/multi-thread-copying.c +index 06cdb8e..9267545 100644 +--- a/copy/multi-thread-copying.c ++++ b/copy/multi-thread-copying.c +@@ -166,6 +166,62 @@ decrease_queue_size (struct worker *worker, size_t len) + worker->queue_size -= len; + } + ++/* Using the extents map 'exts', check if the region ++ * [offset..offset+len-1] intersects only with zero extents. ++ * ++ * The invariant for '*i' is always an extent which starts before or ++ * equal to the current offset. ++ */ ++static bool ++only_zeroes (const extent_list exts, size_t *i, ++ uint64_t offset, unsigned len) ++{ ++ size_t j; ++ ++ /* Invariant. */ ++ assert (*i < exts.len); ++ assert (exts.ptr[*i].offset <= offset); ++ ++ /* Update the invariant. Search for the last possible extent in the ++ * list which is <= offset. ++ */ ++ for (j = *i + 1; j < exts.len; ++j) { ++ if (exts.ptr[j].offset <= offset) ++ *i = j; ++ else ++ break; ++ } ++ ++ /* Check invariant again. */ ++ assert (*i < exts.len); ++ assert (exts.ptr[*i].offset <= offset); ++ ++ /* If *i is not the last extent, then the next extent starts ++ * strictly beyond our current offset. ++ */ ++ assert (*i == exts.len - 1 || exts.ptr[*i + 1].offset > offset); ++ ++ /* Search forward, look for any non-zero extents overlapping the region. */ ++ for (j = *i; j < exts.len; ++j) { ++ uint64_t start, end; ++ ++ /* [start..end-1] is the current extent. */ ++ start = exts.ptr[j].offset; ++ end = exts.ptr[j].offset + exts.ptr[j].length; ++ ++ assert (end > offset); ++ ++ if (start >= offset + len) ++ break; ++ ++ /* Non-zero extent covering this region => test failed. */ ++ if (!exts.ptr[j].zero) ++ return false; ++ } ++ ++ return true; ++} ++ + /* There are 'threads' worker threads, each copying work ranges from + * src to dst until there are no more work ranges. + */ +@@ -177,7 +233,10 @@ worker_thread (void *wp) + extent_list exts = empty_vector; + + while (get_next_offset (&offset, &count)) { +- size_t i; ++ struct command *command; ++ size_t extent_index; ++ bool is_zeroing = false; ++ uint64_t zeroing_start = 0; /* initialized to avoid bogus GCC warning */ + + assert (0 < count && count <= THREAD_WORK_SIZE); + if (extents) +@@ -185,52 +244,64 @@ worker_thread (void *wp) + else + default_get_extents (src, w->index, offset, count, &exts); + +- for (i = 0; i < exts.len; ++i) { +- struct command *command; +- size_t len; ++ extent_index = 0; // index into extents array used to optimize only_zeroes ++ while (count) { ++ const size_t len = MIN (count, request_size); + +- if (exts.ptr[i].zero) { ++ if (only_zeroes (exts, &extent_index, offset, len)) { + /* The source is zero so we can proceed directly to skipping, +- * fast zeroing, or writing zeroes at the destination. ++ * fast zeroing, or writing zeroes at the destination. Defer ++ * zeroing so we can send it as a single large command. + */ +- command = create_command (exts.ptr[i].offset, exts.ptr[i].length, +- true, w); +- fill_dst_range_with_zeroes (command); ++ if (!is_zeroing) { ++ is_zeroing = true; ++ zeroing_start = offset; ++ } + } +- + else /* data */ { +- /* As the extent might be larger than permitted for a single +- * command, we may have to split this into multiple read +- * requests. +- */ +- while (exts.ptr[i].length > 0) { +- len = exts.ptr[i].length; +- if (len > request_size) +- len = request_size; +- +- command = create_command (exts.ptr[i].offset, len, +- false, w); +- +- wait_for_request_slots (w); +- +- /* NOTE: Must increase the queue size after waiting. */ +- increase_queue_size (w, len); +- +- /* Begin the asynch read operation. */ +- src->ops->asynch_read (src, command, +- (nbd_completion_callback) { +- .callback = finished_read, +- .user_data = command, +- }); +- +- exts.ptr[i].offset += len; +- exts.ptr[i].length -= len; ++ /* If we were in the middle of deferred zeroing, do it now. */ ++ if (is_zeroing) { ++ /* Note that offset-zeroing_start can never exceed ++ * THREAD_WORK_SIZE, so there is no danger of overflowing ++ * size_t. ++ */ ++ command = create_command (zeroing_start, offset-zeroing_start, ++ true, w); ++ fill_dst_range_with_zeroes (command); ++ is_zeroing = false; + } ++ ++ /* Issue the asynchronous read command. */ ++ command = create_command (offset, len, false, w); ++ ++ wait_for_request_slots (w); ++ ++ /* NOTE: Must increase the queue size after waiting. */ ++ increase_queue_size (w, len); ++ ++ /* Begin the asynch read operation. */ ++ src->ops->asynch_read (src, command, ++ (nbd_completion_callback) { ++ .callback = finished_read, ++ .user_data = command, ++ }); + } + +- offset += count; +- count = 0; +- } /* for extents */ ++ offset += len; ++ count -= len; ++ } /* while (count) */ ++ ++ /* If we were in the middle of deferred zeroing, do it now. */ ++ if (is_zeroing) { ++ /* Note that offset-zeroing_start can never exceed ++ * THREAD_WORK_SIZE, so there is no danger of overflowing ++ * size_t. ++ */ ++ command = create_command (zeroing_start, offset - zeroing_start, ++ true, w); ++ fill_dst_range_with_zeroes (command); ++ is_zeroing = false; ++ } + } + + /* Wait for in flight NBD requests to finish. */ +diff --git a/copy/nbdcopy.pod b/copy/nbdcopy.pod +index fd10f7c..f06d112 100644 +--- a/copy/nbdcopy.pod ++++ b/copy/nbdcopy.pod +@@ -182,8 +182,9 @@ Set the maximum number of requests in flight per NBD connection. + =item B<--sparse=>N + + Detect all zero blocks of size N (bytes) and make them sparse on the +-output. You can also turn off sparse detection using S>. +-The default is 4096 bytes. ++output. You can also turn off sparse detection using S>. The ++default is 4096 bytes, or the destination preferred block size, ++whichever is larger. + + =item B<--synchronous> + +-- +2.31.1 + diff --git a/SOURCES/0008-dump-Add-another-example-to-the-manual.patch b/SOURCES/0008-dump-Add-another-example-to-the-manual.patch new file mode 100644 index 0000000..1a4eb5b --- /dev/null +++ b/SOURCES/0008-dump-Add-another-example-to-the-manual.patch @@ -0,0 +1,29 @@ +From 5d21b00dbdd1e1a04317bf16afb8f4d2ceaa470f Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 2 Jul 2022 17:12:46 +0100 +Subject: [PATCH] dump: Add another example to the manual + +(cherry picked from commit be3768b077c9542aba34eb821016c36f31d234af) +--- + dump/nbddump.pod | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/dump/nbddump.pod b/dump/nbddump.pod +index 5d7864d..656a965 100644 +--- a/dump/nbddump.pod ++++ b/dump/nbddump.pod +@@ -57,6 +57,11 @@ For example, to dump out a qcow2 file as raw data: + + nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ] + ++To dump out an empty floppy disk created by L: ++ ++ mkdir /var/tmp/empty ++ nbddump -- [ nbdkit floppy /var/tmp/empty ] ++ + Note that S> are separate parameters, and must be + surrounded by spaces. C<--> separates nbddump parameters from + subprocess parameters. +-- +2.31.1 + diff --git a/SOURCES/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch b/SOURCES/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch new file mode 100644 index 0000000..e33ee16 --- /dev/null +++ b/SOURCES/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch @@ -0,0 +1,93 @@ +From a432e773e0cdc24cb27ccdda4111744ea2c3b819 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 27 Jul 2022 17:08:14 +0100 +Subject: [PATCH] lib/crypto: Use GNUTLS_NO_SIGNAL if available + +libnbd has long used MSG_NOSIGNAL to avoid receiving SIGPIPE if we +accidentally write on a closed socket, which is a nice alternative to +using a SIGPIPE signal handler. However with TLS connections, gnutls +did not use this flag and so programs using libnbd + TLS would receive +SIGPIPE in some situations, notably if the server closed the +connection abruptly while we were trying to write something. + +GnuTLS 3.4.2 introduces GNUTLS_NO_SIGNAL which does the same thing. +Use this flag if available. + +RHEL 7 has an older gnutls which lacks this flag. To avoid qemu-nbd +interop tests failing (rarely, but more often with a forthcoming +change to TLS shutdown behaviour), register a SIGPIPE signal handler +in the test if the flag is missing. +--- + configure.ac | 15 +++++++++++++++ + interop/interop.c | 10 ++++++++++ + lib/crypto.c | 7 ++++++- + 3 files changed, 31 insertions(+), 1 deletion(-) + +diff --git a/configure.ac b/configure.ac +index 49ca8ab..6bd9e1b 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -179,6 +179,21 @@ AS_IF([test "$GNUTLS_LIBS" != ""],[ + gnutls_session_set_verify_cert \ + gnutls_transport_is_ktls_enabled \ + ]) ++ AC_MSG_CHECKING([if gnutls has GNUTLS_NO_SIGNAL]) ++ AC_COMPILE_IFELSE( ++ [AC_LANG_PROGRAM([ ++ #include ++ gnutls_session_t session; ++ ], [ ++ gnutls_init(&session, GNUTLS_CLIENT|GNUTLS_NO_SIGNAL); ++ ]) ++ ], [ ++ AC_MSG_RESULT([yes]) ++ AC_DEFINE([HAVE_GNUTLS_NO_SIGNAL], [1], ++ [GNUTLS_NO_SIGNAL found at compile time]) ++ ], [ ++ AC_MSG_RESULT([no]) ++ ]) + LIBS="$old_LIBS" + ]) + +diff --git a/interop/interop.c b/interop/interop.c +index b41f3ca..036545b 100644 +--- a/interop/interop.c ++++ b/interop/interop.c +@@ -84,6 +84,16 @@ main (int argc, char *argv[]) + REQUIRES + #endif + ++ /* Ignore SIGPIPE. We only need this for GnuTLS < 3.4.2, since ++ * newer GnuTLS has the GNUTLS_NO_SIGNAL flag which adds ++ * MSG_NOSIGNAL to each write call. ++ */ ++#if !HAVE_GNUTLS_NO_SIGNAL ++#if TLS ++ signal (SIGPIPE, SIG_IGN); ++#endif ++#endif ++ + /* Create a large sparse temporary file. */ + #ifdef NEEDS_TMPFILE + int fd = mkstemp (TMPFILE); +diff --git a/lib/crypto.c b/lib/crypto.c +index 1272888..ca9520e 100644 +--- a/lib/crypto.c ++++ b/lib/crypto.c +@@ -588,7 +588,12 @@ nbd_internal_crypto_create_session (struct nbd_handle *h, + gnutls_psk_client_credentials_t pskcreds = NULL; + gnutls_certificate_credentials_t xcreds = NULL; + +- err = gnutls_init (&session, GNUTLS_CLIENT|GNUTLS_NONBLOCK); ++ err = gnutls_init (&session, ++ GNUTLS_CLIENT | GNUTLS_NONBLOCK ++#if HAVE_GNUTLS_NO_SIGNAL ++ | GNUTLS_NO_SIGNAL ++#endif ++ ); + if (err < 0) { + set_error (errno, "gnutls_init: %s", gnutls_strerror (err)); + return NULL; +-- +2.31.1 + diff --git a/SOURCES/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch b/SOURCES/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch new file mode 100644 index 0000000..cc91332 --- /dev/null +++ b/SOURCES/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch @@ -0,0 +1,100 @@ +From 8bbee9c0ff052cf8ab5ba81fd1b67e3c45e7012a Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 27 Jul 2022 16:07:37 +0100 +Subject: [PATCH] lib/crypto.c: Ignore TLS premature termination after write + shutdown + +qemu-nbd doesn't call gnutls_bye to cleanly shut down the connection +after we send NBD_CMD_DISC. When copying from a qemu-nbd server (or +any operation which calls nbd_shutdown) you will see errors like this: + + $ nbdcopy nbds://foo?tls-certificates=/var/tmp/pki null: + nbds://foo?tls-certificates=/var/tmp/pki: nbd_shutdown: gnutls_record_recv: The TLS connection was non-properly terminated. + +Relatedly you may also see: + + nbd_shutdown: gnutls_record_recv: Error in the pull function. + +This commit suppresses the error in the case where we know that we +have shut down writes (which happens after NBD_CMD_DISC has been sent +on the wire). +--- + interop/interop.c | 9 --------- + lib/crypto.c | 17 +++++++++++++++++ + lib/internal.h | 1 + + 3 files changed, 18 insertions(+), 9 deletions(-) + +diff --git a/interop/interop.c b/interop/interop.c +index 036545b..cce9407 100644 +--- a/interop/interop.c ++++ b/interop/interop.c +@@ -226,19 +226,10 @@ main (int argc, char *argv[]) + + /* XXX In future test more operations here. */ + +-#if !TLS +- /* XXX qemu doesn't shut down the connection nicely (using +- * gnutls_bye) and because of this the following call will fail +- * with: +- * +- * nbd_shutdown: gnutls_record_recv: The TLS connection was +- * non-properly terminated. +- */ + if (nbd_shutdown (nbd, 0) == -1) { + fprintf (stderr, "%s\n", nbd_get_error ()); + exit (EXIT_FAILURE); + } +-#endif + + nbd_close (nbd); + +diff --git a/lib/crypto.c b/lib/crypto.c +index ca9520e..aa5d820 100644 +--- a/lib/crypto.c ++++ b/lib/crypto.c +@@ -187,6 +187,22 @@ tls_recv (struct nbd_handle *h, struct socket *sock, void *buf, size_t len) + errno = EAGAIN; + return -1; + } ++ if (h->tls_shut_writes && ++ (r == GNUTLS_E_PULL_ERROR || r == GNUTLS_E_PREMATURE_TERMINATION)) { ++ /* qemu-nbd doesn't call gnutls_bye to cleanly shut down the ++ * connection after we send NBD_CMD_DISC, instead it simply ++ * closes the connection. On the client side we see ++ * "gnutls_record_recv: The TLS connection was non-properly ++ * terminated" or "gnutls_record_recv: Error in the pull ++ * function.". ++ * ++ * If we see these errors after we shut down the write side ++ * (h->tls_shut_writes), which happens after we have sent ++ * NBD_CMD_DISC on the wire, downgrade them to a debug message. ++ */ ++ debug (h, "gnutls_record_recv: %s", gnutls_strerror (r)); ++ return 0; /* EOF */ ++ } + set_error (0, "gnutls_record_recv: %s", gnutls_strerror (r)); + errno = EIO; + return -1; +@@ -234,6 +250,7 @@ tls_shut_writes (struct nbd_handle *h, struct socket *sock) + return false; + if (r != 0) + debug (h, "ignoring gnutls_bye failure: %s", gnutls_strerror (r)); ++ h->tls_shut_writes = true; + return sock->u.tls.oldsock->ops->shut_writes (h, sock->u.tls.oldsock); + } + +diff --git a/lib/internal.h b/lib/internal.h +index 6aaced3..f1b4c63 100644 +--- a/lib/internal.h ++++ b/lib/internal.h +@@ -307,6 +307,7 @@ struct nbd_handle { + struct command *reply_cmd; + + bool disconnect_request; /* True if we've queued NBD_CMD_DISC */ ++ bool tls_shut_writes; /* Used by lib/crypto.c to track disconnect. */ + }; + + struct meta_context { +-- +2.31.1 + diff --git a/SOURCES/copy-patches.sh b/SOURCES/copy-patches.sh new file mode 100755 index 0000000..65d5933 --- /dev/null +++ b/SOURCES/copy-patches.sh @@ -0,0 +1,55 @@ +#!/bin/bash - + +set -e + +# Maintainer script to copy patches from the git repo to the current +# directory. Use it like this: +# ./copy-patches.sh + +rhel_version=9.1 + +# Check we're in the right directory. +if [ ! -f libnbd.spec ]; then + echo "$0: run this from the directory containing 'libnbd.spec'" + exit 1 +fi + +git_checkout=$HOME/d/libnbd-rhel-$rhel_version +if [ ! -d $git_checkout ]; then + echo "$0: $git_checkout does not exist" + echo "This script is only for use by the maintainer when preparing a" + echo "libnbd release on RHEL." + exit 1 +fi + +# Get the base version of libnbd. +version=`grep '^Version:' libnbd.spec | awk '{print $2}'` +tag="v$version" + +# Remove any existing patches. +git rm -f [0-9]*.patch ||: +rm -f [0-9]*.patch + +# Get the patches. +(cd $git_checkout; rm -f [0-9]*.patch; git format-patch -N $tag) +mv $git_checkout/[0-9]*.patch . + +# Remove any not to be applied. +rm -f *NOT-FOR-RPM*.patch + +# Add the patches. +git add [0-9]*.patch + +# Print out the patch lines. +echo +echo "--- Copy the following text into libnbd.spec file" +echo + +echo "# Patches." +for f in [0-9]*.patch; do + n=`echo $f | awk -F- '{print $1}'` + echo "Patch$n: $f" +done + +echo +echo "--- End of text" diff --git a/SOURCES/libnbd-1.12.6.tar.gz.sig b/SOURCES/libnbd-1.12.6.tar.gz.sig new file mode 100644 index 0000000..4726053 --- /dev/null +++ b/SOURCES/libnbd-1.12.6.tar.gz.sig @@ -0,0 +1,17 @@ +-----BEGIN PGP SIGNATURE----- + +iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmLhNK4RHHJpY2hAYW5u +ZXhpYS5vcmcACgkQkXOPc+G3aKDzmxAArFrR/cOyqyGZXuYORFRVi7AobCjum4dP +A93R43shnSXXB1CTww5O+LjIghSLs4TEQAOcmcsjsE98X2cz0BuW6gIfGxTpN3WP +fGPDlvezLXGo5zX5WGFkP6oQY97TuGHXKNxStZtWRtDNfrWPWJQuwlm5GSIHdYYr +dFssmDNtIoh/zQz2www9JKspMfehFbTGZswtRjfDwa2Pl69cMy3pH/k4EZZnDx9n +tguzQHOapJJx8RkIwUwFirCBOwdVNbLX+KrGroLcB6MjO6Uhh2C/iYUQDM/xX2r6 +1SugssAmXwZ4/RIDtwBLdQdEoNjSAV7OW1yizTl5P9qFkiPr+Lpnpt4gci9bTZKK +oIy0RtgJOgW5R2tuRlMXkx/7kcGjhUb3Zbux7d7dgrYq16FUPC1dFgub7WSPSqWe ++17iUD+n3NO2MHtR215nKDjuPR59wnvATO6QS+InOb4imyf47Ic7TeEvVuDhb+M3 ++DvIor2WyXWX9kO165Fx9jAicgZJt2L1UvKM1GzGjwBdL0GYbCFltzzhyi4Pd/7x +ijV2QYbyOLXYEpsgmKYrT6MwwXUAye2PkXSv8MaOs3IiFz0bVy3Bfgx4UYNbKo1x +zwVGtIBz39tXpgyS7+F9rvQILVmENGmyTx+GXrv/lE1mFmTt/EEyf+iHMSN1lIkt +59o9LBpOajI= +=idnt +-----END PGP SIGNATURE----- diff --git a/SPECS/libnbd.spec b/SPECS/libnbd.spec new file mode 100644 index 0000000..06155a2 --- /dev/null +++ b/SPECS/libnbd.spec @@ -0,0 +1,690 @@ +# Do this until the feature is fixed in Fedora. +%undefine _package_note_flags + +# If we should verify tarball signature with GPGv2. +%global verify_tarball_signature 1 + +# If there are patches which touch autotools files, set this to 1. +%global patches_touch_autotools 1 + +# The source directory. +%global source_directory 1.12-stable + +Name: libnbd +Version: 1.12.6 +Release: 1%{?dist} +Summary: NBD client library in userspace + +License: LGPLv2+ +URL: https://gitlab.com/nbdkit/libnbd + +Source0: http://libguestfs.org/download/libnbd/%{source_directory}/%{name}-%{version}.tar.gz +Source1: http://libguestfs.org/download/libnbd/%{source_directory}/%{name}-%{version}.tar.gz.sig +# Keyring used to verify tarball signature. This contains the single +# key from here: +# https://pgp.key-server.io/pks/lookup?search=rjones%40redhat.com&fingerprint=on&op=vindex +Source2: libguestfs.keyring + +# Maintainer script which helps with handling patches. +Source3: copy-patches.sh + +# Patches are stored in the upstream repository: +# https://gitlab.com/nbdkit/libnbd/-/commits/rhel-9.1/ + +# Patches. +Patch0001: 0001-Add-nbddump-tool.patch +Patch0002: 0002-dump-Visually-separate-columns-0-7-and-8-15.patch +Patch0003: 0003-dump-Fix-build-on-i686.patch +Patch0004: 0004-dump-Fix-tests-on-Debian-10.patch +Patch0005: 0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch +Patch0006: 0006-copy-Store-the-preferred-block-size-in-the-operation.patch +Patch0007: 0007-copy-Use-preferred-block-size-for-copying.patch +Patch0008: 0008-dump-Add-another-example-to-the-manual.patch +Patch0009: 0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch +Patch0010: 0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch + +%if 0%{patches_touch_autotools} +BuildRequires: autoconf, automake, libtool +%endif + +%if 0%{verify_tarball_signature} +BuildRequires: gnupg2 +%endif + +# For the core library. +BuildRequires: gcc +BuildRequires: make +BuildRequires: /usr/bin/pod2man +BuildRequires: gnutls-devel +BuildRequires: libxml2-devel + +# For nbdfuse. +BuildRequires: fuse3, fuse3-devel + +# For the Python 3 bindings. +BuildRequires: python3-devel + +# For the OCaml bindings. +BuildRequires: ocaml +BuildRequires: ocaml-findlib-devel +BuildRequires: ocaml-ocamldoc + +# Only for building the examples. +BuildRequires: glib2-devel + +# For bash-completion. +BuildRequires: bash-completion + +# Only for running the test suite. +BuildRequires: coreutils +BuildRequires: gcc-c++ +BuildRequires: gnutls-utils +BuildRequires: iproute +BuildRequires: jq +%if !0%{?rhel} +BuildRequires: nbd +%endif +BuildRequires: util-linux + +# On RHEL, maybe even in Fedora in future, we do not build qemu-img or +# nbdkit for i686. These are only needed for the test suite so make +# them optional. This reduces our test exposure on 32 bit platforms, +# although there is still Fedora/armv7 and some upstream testing. +%ifnarch %{ix86} +BuildRequires: qemu-img +BuildRequires: nbdkit +BuildRequires: nbdkit-data-plugin +BuildRequires: nbdkit-eval-plugin +BuildRequires: nbdkit-memory-plugin +BuildRequires: nbdkit-null-plugin +BuildRequires: nbdkit-pattern-plugin +BuildRequires: nbdkit-sh-plugin +BuildRequires: nbdkit-sparse-random-plugin +%endif + + +%description +NBD — Network Block Device — is a protocol for accessing Block Devices +(hard disks and disk-like things) over a Network. + +This is the NBD client library in userspace, a simple library for +writing NBD clients. + +The key features are: + + * Synchronous and asynchronous APIs, both for ease of use and for + writing non-blocking, multithreaded clients. + + * High performance. + + * Minimal dependencies for the basic library. + + * Well-documented, stable API. + + * Bindings in several programming languages. + + +%package devel +Summary: Development headers for %{name} +License: LGPLv2+ and BSD +Requires: %{name}%{?_isa} = %{version}-%{release} + + +%description devel +This package contains development headers for %{name}. + + +%package -n ocaml-%{name} +Summary: OCaml language bindings for %{name} +Requires: %{name}%{?_isa} = %{version}-%{release} + + +%description -n ocaml-%{name} +This package contains OCaml language bindings for %{name}. + + +%package -n ocaml-%{name}-devel +Summary: OCaml language development package for %{name} +Requires: ocaml-%{name}%{?_isa} = %{version}-%{release} + + +%description -n ocaml-%{name}-devel +This package contains OCaml language development package for +%{name}. Install this if you want to compile OCaml software which +uses %{name}. + + +%package -n python3-%{name} +Summary: Python 3 bindings for %{name} +Requires: %{name}%{?_isa} = %{version}-%{release} +%{?python_provide:%python_provide python3-%{name}} + +# The Python module happens to be called lib*.so. Don't scan it and +# have a bogus "Provides: libnbdmod.*". +%global __provides_exclude_from ^%{python3_sitearch}/lib.*\\.so + + +%description -n python3-%{name} +python3-%{name} contains Python 3 bindings for %{name}. + + +%package -n nbdfuse +Summary: FUSE support for %{name} +License: LGPLv2+ and BSD +Requires: %{name}%{?_isa} = %{version}-%{release} +Recommends: fuse3 + + +%description -n nbdfuse +This package contains FUSE support for %{name}. + + +%package bash-completion +Summary: Bash tab-completion for %{name} +BuildArch: noarch +Requires: bash-completion >= 2.0 +# Don't use _isa here because it's a noarch package. This dependency +# is just to ensure that the subpackage is updated along with libnbd. +Requires: %{name} = %{version}-%{release} + + +%description bash-completion +Install this package if you want intelligent bash tab-completion +for %{name}. + + +%prep +%if 0%{verify_tarball_signature} +%{gpgverify} --keyring='%{SOURCE2}' --signature='%{SOURCE1}' --data='%{SOURCE0}' +%endif +%autosetup -p1 +%if 0%{patches_touch_autotools} +autoreconf -i +%endif + + +%build +%configure \ + --disable-static \ + --with-tls-priority=@LIBNBD,SYSTEM \ + PYTHON=%{__python3} \ + --enable-python \ + --enable-ocaml \ + --enable-fuse \ + --disable-golang + +make %{?_smp_mflags} + + +%install +%make_install + +# Delete libtool crap. +find $RPM_BUILD_ROOT -name '*.la' -delete + +# Delete the golang man page since we're not distributing the bindings. +rm $RPM_BUILD_ROOT%{_mandir}/man3/libnbd-golang.3* + + +%check +function skip_test () +{ + for f in "$@"; do + rm -f "$f" + echo 'exit 77' > "$f" + chmod +x "$f" + done +} + +# interop/structured-read.sh fails with the old qemu-nbd in Fedora 29, +# so disable it there. +%if 0%{?fedora} <= 29 +skip_test interop/structured-read.sh +%endif + +# interop/interop-qemu-storage-daemon.sh fails in RHEL 9 because of +# this bug in qemu: +# https://lists.nongnu.org/archive/html/qemu-devel/2021-03/threads.html#03544 +%if 0%{?rhel} +skip_test interop/interop-qemu-storage-daemon.sh +%endif + +# All fuse tests fail in Koji with: +# fusermount: entry for fuse/test-*.d not found in /etc/mtab +# for unknown reasons but probably related to the Koji environment. +skip_test fuse/test-*.sh + +# IPv6 loopback connections fail in Koji. +make -C tests connect-tcp6 ||: +skip_test tests/connect-tcp6 + +make %{?_smp_mflags} check || { + for f in $(find -name test-suite.log); do + echo + echo "==== $f ====" + cat $f + done + exit 1 + } + + +%files +%doc README +%license COPYING.LIB +%{_bindir}/nbdcopy +%{_bindir}/nbddump +%{_bindir}/nbdinfo +%{_libdir}/libnbd.so.* +%{_mandir}/man1/nbdcopy.1* +%{_mandir}/man1/nbddump.1* +%{_mandir}/man1/nbdinfo.1* + + +%files devel +%doc TODO examples/*.c +%license examples/LICENSE-FOR-EXAMPLES +%{_includedir}/libnbd.h +%{_libdir}/libnbd.so +%{_libdir}/pkgconfig/libnbd.pc +%{_mandir}/man3/libnbd.3* +%{_mandir}/man1/libnbd-release-notes-1.*.1* +%{_mandir}/man3/libnbd-security.3* +%{_mandir}/man3/nbd_*.3* + + +%files -n ocaml-%{name} +%{_libdir}/ocaml/nbd +%exclude %{_libdir}/ocaml/nbd/*.a +%exclude %{_libdir}/ocaml/nbd/*.cmxa +%exclude %{_libdir}/ocaml/nbd/*.cmx +%exclude %{_libdir}/ocaml/nbd/*.mli +%{_libdir}/ocaml/stublibs/dllmlnbd.so +%{_libdir}/ocaml/stublibs/dllmlnbd.so.owner + + +%files -n ocaml-%{name}-devel +%doc ocaml/examples/*.ml +%license ocaml/examples/LICENSE-FOR-EXAMPLES +%{_libdir}/ocaml/nbd/*.a +%{_libdir}/ocaml/nbd/*.cmxa +%{_libdir}/ocaml/nbd/*.cmx +%{_libdir}/ocaml/nbd/*.mli +%{_mandir}/man3/libnbd-ocaml.3* +%{_mandir}/man3/NBD.3* +%{_mandir}/man3/NBD.*.3* + + +%files -n python3-%{name} +%{python3_sitearch}/libnbdmod*.so +%{python3_sitearch}/nbd.py +%{python3_sitearch}/nbdsh.py +%{python3_sitearch}/__pycache__/nbd*.py* +%{_bindir}/nbdsh +%{_mandir}/man1/nbdsh.1* + + +%files -n nbdfuse +%{_bindir}/nbdfuse +%{_mandir}/man1/nbdfuse.1* + + +%files bash-completion +%dir %{_datadir}/bash-completion/completions +%{_datadir}/bash-completion/completions/nbdcopy +%{_datadir}/bash-completion/completions/nbddump +%{_datadir}/bash-completion/completions/nbdfuse +%{_datadir}/bash-completion/completions/nbdinfo +%{_datadir}/bash-completion/completions/nbdsh + + +%changelog +* Thu Jul 28 2022 Richard W.M. Jones - 1.12.6-1 +- Rebase to new stable branch version 1.12.6 + resolves: rhbz#2059288 +- New tool: nbddump +- nbdcopy: Use preferred block size for copying + related: rhbz#2047660 +- Fix remote TLS failures + resolves: rhbz#2111524 + (and 2111813) + +* Thu Feb 10 2022 Richard W.M. Jones - 1.10.5-1 +- Rebase to new stable branch version 1.10.5 + resolves: rhbz#2011708 +- Map uint32_t to OCaml int64 to avoid signedness problems + resolves: rhbz#2040610 +- CVE-2022-0485 nbdcopy destination image corruption +- New upstream API to control initialization of pread buffer + resolves: rhbz#2046194 + +* Mon Aug 09 2021 Mohan Boddu - 1.8.2-3 +- Rebuilt for IMA sigs, glibc 2.34, aarch64 flags + Related: rhbz#1991688 + +* Fri Jul 30 2021 Richard W.M. Jones - 1.8.2-2 +- Fix nbdcopy progress bar. +- Add nbdinfo --map --totals and --can/--is options. + resolves: rhbz#1950630 + +* Sat Jul 03 2021 Richard W.M. Jones - 1.8.2-1 +- New upstream stable version 1.8.2. + +* Wed Jun 23 2021 Richard W.M. Jones - 1.8.1-2 +- Bump and rebuild + resolves: rhbz#1975316 + +* Fri Jun 11 2021 Richard W.M. Jones - 1.8.1-1 +- New upstream stable version 1.8.1. + +* Mon Jun 07 2021 Richard W.M. Jones - 1.8.0-1 +- New upstream version 1.8.0. + +* Fri Jun 04 2021 Python Maint - 1.7.12-2 +- Rebuilt for Python 3.10 + +* Sat May 29 2021 Richard W.M. Jones - 1.7.12-1 +- New upstream version 1.7.12. + +* Thu May 20 2021 Richard W.M. Jones - 1.7.11-1 +- New upstream version 1.7.11. + +* Fri May 14 2021 Richard W.M. Jones - 1.7.10-1 +- New upstream version 1.7.10. + +* Thu Apr 29 2021 Richard W.M. Jones - 1.7.9-1 +- New upstream version 1.7.9. +- Switch to fuse3. +- Make nbdfuse package recommend fuse3 (to get fusermount3). + +* Sat Apr 24 2021 Richard W.M. Jones - 1.7.8-1 +- New upstream development version 1.7.8. + +* Sat Apr 10 2021 Richard W.M. Jones - 1.7.7-1 +- New upstream development version 1.7.7. +- +BR iproute +- Add skip_test helper function. +- Skip connect-tcp6 test which fails under Koji. + +* Thu Apr 08 2021 Richard W.M. Jones - 1.7.6-1 +- New upstream development version 1.7.6. + +* Sat Apr 03 2021 Richard W.M. Jones - 1.7.5-1 +- New upstream development version 1.7.5. + +* Mon Mar 15 2021 Richard W.M. Jones - 1.7.4-1 +- New upstream development version 1.7.4. + +* Mon Mar 15 2021 Richard W.M. Jones - 1.7.3-3 +- Update documentation for CVE-2021-20286. +- Workaround broken interop/interop-qemu-storage-daemon.sh test in RHEL 9. + +* Thu Mar 4 2021 Richard W.M. Jones - 1.7.3-2 +- Add fix for nbdkit test suite. + +* Tue Mar 2 2021 Richard W.M. Jones - 1.7.3-1 +- New upstream version 1.7.3. + +* Mon Mar 1 2021 Richard W.M. Jones - 1.7.2-3 +- OCaml 4.12.0 build + +* Wed Feb 24 2021 Richard W.M. Jones - 1.7.2-2 +- Disable nbd BR on RHEL. + +* Mon Feb 22 2021 Richard W.M. Jones - 1.7.2-1 +- New upstream version 1.7.2. + +* Fri Jan 29 2021 Richard W.M. Jones - 1.7.1-6 +- Disable BR qemu-img on i686. + +* Thu Jan 28 2021 Richard W.M. Jones - 1.7.1-3 +- Disable BR nbdkit on i686 because it breaks ELN/RHEL 9. + +* Tue Jan 26 2021 Fedora Release Engineering - 1.7.1-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_34_Mass_Rebuild + +* Wed Jan 20 2021 Richard W.M. Jones - 1.7.1-1 +- New upstream development version 1.7.1. + +* Thu Jan 07 2021 Richard W.M. Jones - 1.6.0-1 +- New upstream stable version 1.6.0. + +* Tue Dec 08 2020 Richard W.M. Jones - 1.5.9-1 +- New upstream development version 1.5.9. + +* Thu Dec 03 2020 Richard W.M. Jones - 1.5.8-1 +- New upstream development version 1.5.8. +- Unify Fedora and RHEL spec files. + +* Wed Nov 25 2020 Richard W.M. Jones - 1.5.7-1 +- New upstream development version 1.5.7. +- Add some more test suite buildrequires lines. +- Fix bogus date in changelog. + +* Thu Nov 12 2020 Richard W.M. Jones - 1.5.6-1 +- New upstream development version 1.5.6. + +* Mon Nov 02 2020 Richard W.M. Jones - 1.5.5-1 +- New upstream development version 1.5.5. + +* Mon Oct 05 2020 Richard W.M. Jones - 1.5.4-1 +- New upstream development version 1.5.4. +- More OCaml man pages. + +* Sat Sep 26 2020 Richard W.M. Jones - 1.5.3-1 +- New upstream development version 1.5.3. + +* Thu Sep 10 2020 Richard W.M. Jones - 1.5.2-1 +- New upstream development version 1.5.2. + +* Tue Sep 08 2020 Richard W.M. Jones - 1.5.1-1 +- New upstream development version 1.5.1. + +* Tue Sep 01 2020 Richard W.M. Jones - 1.4.0-2 +- OCaml 4.11.1 rebuild + +* Tue Aug 25 2020 Richard W.M. Jones - 1.4.0-1 +- New stable release 1.4.0. + +* Fri Aug 21 2020 Richard W.M. Jones - 1.3.12-3 +- Bump release and rebuild. + +* Fri Aug 21 2020 Richard W.M. Jones - 1.3.12-2 +- OCaml 4.11.0 rebuild + +* Thu Aug 20 2020 Richard W.M. Jones - 1.3.12-1 +- New upstream version 1.3.12. + +* Thu Aug 6 2020 Richard W.M. Jones - 1.3.11-1 +- New upstream version 1.3.11. + +* Tue Aug 4 2020 Richard W.M. Jones - 1.3.10-1 +- New upstream version 1.3.10. + +* Wed Jul 29 2020 Richard W.M. Jones - 1.3.9-3 +- Bump and rebuild. + +* Tue Jul 28 2020 Fedora Release Engineering - 1.3.9-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_33_Mass_Rebuild + +* Tue Jul 21 2020 Richard W.M. Jones - 1.3.9-1 +- New upstream version 1.3.9. +- New tool: nbdinfo. + +* Fri Jul 17 2020 Richard W.M. Jones - 1.3.8-2 +- New upstream version 1.3.8. +- New tool: nbdcopy +- Add upstream patch to fix compilation with glibc from Rawhide. + +* Tue May 26 2020 Miro Hrončok - 1.3.7-3 +- Rebuilt for Python 3.9 + +* Mon May 04 2020 Richard W.M. Jones - 1.3.7-2 +- OCaml 4.11.0+dev2-2020-04-22 rebuild + +* Thu Apr 23 2020 Richard W.M. Jones - 1.3.7-1 +- New upstream version 1.3.7. + +* Tue Apr 21 2020 Richard W.M. Jones - 1.3.6-5 +- OCaml 4.11.0 pre-release attempt 2 + +* Fri Apr 17 2020 Richard W.M. Jones - 1.3.6-4 +- OCaml 4.11.0 pre-release +- Add upstream patch to fix one of the tests that fails on slow machines. + +* Thu Apr 02 2020 Richard W.M. Jones - 1.3.6-2 +- Update all OCaml dependencies for RPM 4.16. + +* Tue Mar 31 2020 Richard W.M. Jones - 1.3.6-1 +- New upstream development version 1.3.6. +- Golang bindings are contained in this release but not distributed. + +* Wed Mar 11 2020 Richard W.M. Jones - 1.3.5-2 +- Fix bogus runtime Requires of new bash-completion package. + +* Tue Mar 10 2020 Richard W.M. Jones - 1.3.5-1 +- New upstream development version 1.3.5. +- Add new bash-completion subpackage. + +* Sat Feb 29 2020 Richard W.M. Jones - 1.3.4-1 +- New upstream development version 1.3.4. + +* Wed Feb 26 2020 Richard W.M. Jones - 1.3.3-2 +- OCaml 4.10.0 final. + +* Wed Feb 05 2020 Richard W.M. Jones - 1.3.3-1 +- New upstream development version 1.3.3. + +* Thu Jan 30 2020 Richard W.M. Jones - 1.3.2-1 +- New upstream development version 1.3.2. + +* Wed Jan 29 2020 Fedora Release Engineering - 1.3.1-5 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_32_Mass_Rebuild + +* Sun Jan 19 2020 Richard W.M. Jones - 1.3.1-4 +- Bump release and rebuild. + +* Sun Jan 19 2020 Richard W.M. Jones - 1.3.1-3 +- OCaml 4.10.0+beta1 rebuild. + +* Thu Dec 12 2019 Richard W.M. Jones - 1.3.1-2 +- Rebuild for OCaml 4.09.0. + +* Tue Dec 03 2019 Richard W.M. Jones - 1.3.1-1 +- New upstream development version 1.3.1. + +* Wed Nov 27 2019 Richard W.M. Jones - 1.2.0-2 +- Use gpgverify macro instead of explicit gpgv2 command. + +* Thu Nov 14 2019 Richard W.M. Jones - 1.2.0-1 +- New stable release 1.2.0 + +* Sat Nov 09 2019 Richard W.M. Jones - 1.1.9-1 +- New upstream version 1.1.9. +- Add new nbdkit-release-notes-1.2(1) man page. + +* Wed Nov 06 2019 Richard W.M. Jones - 1.1.8-1 +- New upstream version 1.1.8. + +* Thu Oct 24 2019 Richard W.M. Jones - 1.1.7-1 +- New upstream version 1.1.7. + +* Sat Oct 19 2019 Richard W.M. Jones - 1.1.6-1 +- New upstream version 1.1.6. + +* Sat Oct 12 2019 Richard W.M. Jones - 1.1.5-1 +- New upstream version 1.1.5. +- New tool and subpackage nbdfuse. + +* Wed Oct 9 2019 Richard W.M. Jones - 1.1.4-1 +- New upstream version 1.1.4. +- Contains fix for remote code execution vulnerability. +- Add new libnbd-security(3) man page. + +* Tue Oct 1 2019 Richard W.M. Jones - 1.1.3-1 +- New upstream version 1.1.3. + +* Tue Sep 17 2019 Richard W.M. Jones - 1.1.2-1 +- New upstream version 1.1.2. +- Remove patches which are upstream. +- Contains fix for NBD Protocol Downgrade Attack (CVE-2019-14842). + +* Thu Sep 12 2019 Richard W.M. Jones - 1.1.1-2 +- Add upstream patch to fix nbdsh (for nbdkit tests). + +* Sun Sep 08 2019 Richard W.M. Jones - 1.1.1-1 +- New development version 1.1.1. + +* Wed Aug 28 2019 Richard W.M. Jones - 1.0.0-1 +- New upstream version 1.0.0. + +* Wed Aug 21 2019 Miro Hrončok - 0.9.9-2 +- Rebuilt for Python 3.8 + +* Wed Aug 21 2019 Richard W.M. Jones - 0.9.9-1 +- New upstream version 0.9.9. + +* Wed Aug 21 2019 Richard W.M. Jones - 0.9.8-4 +- Fix nbdkit dependencies so we're actually running the tests. +- Add glib2-devel BR so we build the glib main loop example. +- Add upstream patch to fix test error: + nbd_connect_unix: getlogin: No such device or address +- Fix test failure on 32 bit. + +* Tue Aug 20 2019 Richard W.M. Jones - 0.9.8-3 +- Bump and rebuild to fix releng brokenness. + https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/message/2LIDI33G3IEIPYSCCIP6WWKNHY7XZJGQ/ + +* Mon Aug 19 2019 Miro Hrončok - 0.9.8-2 +- Rebuilt for Python 3.8 + +* Thu Aug 15 2019 Richard W.M. Jones - 0.9.8-1 +- New upstream version 0.9.8. +- Package the new nbd_*(3) man pages. + +* Mon Aug 5 2019 Richard W.M. Jones - 0.9.7-1 +- New upstream version 0.9.7. +- Add libnbd-ocaml(3) man page. + +* Sat Aug 3 2019 Richard W.M. Jones - 0.9.6-2 +- Add all upstream patches since 0.9.6 was released. +- Package the ocaml bindings into a subpackage. + +* Tue Jul 30 2019 Richard W.M. Jones - 0.9.6-1 +- New upstream verison 0.9.6. + +* Fri Jul 26 2019 Richard W.M. Jones - 0.1.9-1 +- New upstream version 0.1.9. + +* Thu Jul 25 2019 Fedora Release Engineering - 0.1.8-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_31_Mass_Rebuild + +* Wed Jul 17 2019 Richard W.M. Jones - 0.1.8-1 +- New upstream version 0.1.8. + +* Tue Jul 16 2019 Richard W.M. Jones - 0.1.7-1 +- New upstream version 0.1.7. + +* Wed Jul 3 2019 Richard W.M. Jones - 0.1.6-1 +- New upstream version 0.1.6. + +* Thu Jun 27 2019 Richard W.M. Jones - 0.1.5-1 +- New upstream version 0.1.5. + +* Sun Jun 09 2019 Richard W.M. Jones - 0.1.4-1 +- New upstream version 0.1.4. + +* Sun Jun 2 2019 Richard W.M. Jones - 0.1.2-2 +- Enable libxml2 for NBD URI support. + +* Thu May 30 2019 Richard W.M. Jones - 0.1.2-1 +- New upstream version 0.1.2. + +* Tue May 28 2019 Richard W.M. Jones - 0.1.1-1 +- Fix license in man pages and examples. +- Add nbdsh(1) man page. +- Include the signature and keyring even if validation is disabled. +- Update devel subpackage license. +- Fix old FSF address in Python tests. +- Filter Python provides. +- Remove executable permission on the tar.gz.sig file. +- Initial release.