You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
273 lines
11 KiB
273 lines
11 KiB
From cc9a507e2372b5b6408964f9c31a3bd526aabf7c Mon Sep 17 00:00:00 2001
|
|
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
Date: Wed, 23 Sep 2020 09:56:27 +0100
|
|
Subject: [PATCH] v2v: vcenter: Implement cookie scripts.
|
|
|
|
For conversions[*] which take longer than 30 minutes it can happen
|
|
that the HTTPS authorization cookie that we fetched from VMware when
|
|
we first connect expires. This can especially happen when there are
|
|
multiple disks, because we may not "touch" (therefore autorenew) the
|
|
second disk while we are doing the long conversion. This can lead to
|
|
failures, some of which are silent: again if there are multiple disks,
|
|
fstrim of the non-system disks can fail silently resulting in the copy
|
|
step taking a very long time.
|
|
|
|
The solution to this is to use the new nbdkit-curl-plugin
|
|
cookie-script feature which allows nbdkit to automatically renew the
|
|
cookie as required.
|
|
|
|
During the conversion or copying steps you may see the cookie being
|
|
autorenewed:
|
|
|
|
nbdkit: curl[3]: debug: curl: running cookie-script
|
|
nbdkit: curl[3]: debug: cookie-script returned cookies
|
|
|
|
This removes the ?user and ?password parameters from Nbdkit_sources.-
|
|
create_curl because they are no longer needed after this change.
|
|
Note for future: if we need to add them back, we must prevent both
|
|
user and cookie_script parameters from being used at the same time,
|
|
because simply having the user parameter will try basic
|
|
authentication, overriding the cookie, which will either fail (no
|
|
password) or run very slowly.
|
|
|
|
This change requires nbdkit >= 1.22 which is checked at runtime only
|
|
if this feature is used.
|
|
|
|
[*] Note here I mean conversions not the total runtime of virt-v2v.
|
|
When doing the copy the cookie does not expire because it is
|
|
continuously auto-renewed by VMware as we continuously access the disk
|
|
(this works differently from systems like Docker where the cookie is
|
|
only valid from the absolute time when it is first created). This
|
|
change also implements the cookie-script logic for copying.
|
|
|
|
(cherry picked from commit 2b9a11743b74ef3716b66a7e395108a26382e331)
|
|
|
|
Notes for cherry pick to RHEL 8.6:
|
|
|
|
We no longer need the session_cookie field inside virt-v2v since it is
|
|
replaced by the cookie script. However it is still needed by
|
|
virt-v2v-copy-to-local. (This utility is removed upstream and in RHEL
|
|
9, but we need to keep it around at least for appearances in RHEL 8.)
|
|
|
|
So when cherry picking I had to retain the get_session_cookie function
|
|
which required also keeping fetch_headers_and_url as it was (not
|
|
dropping headers).
|
|
---
|
|
v2v/nbdkit_sources.ml | 34 ++++++++++++-----
|
|
v2v/nbdkit_sources.mli | 5 +--
|
|
v2v/parse_libvirt_xml.ml | 3 +-
|
|
v2v/vCenter.ml | 80 +++++++++++++++++++++++++++++++---------
|
|
4 files changed, 90 insertions(+), 32 deletions(-)
|
|
|
|
diff --git a/v2v/nbdkit_sources.ml b/v2v/nbdkit_sources.ml
|
|
index 7c177e35..16af5f5c 100644
|
|
--- a/v2v/nbdkit_sources.ml
|
|
+++ b/v2v/nbdkit_sources.ml
|
|
@@ -26,7 +26,6 @@ open Types
|
|
open Utils
|
|
|
|
let nbdkit_min_version = (1, 12, 0)
|
|
-let nbdkit_min_version_string = "1.12.0"
|
|
|
|
type password =
|
|
| NoPassword (* no password option at all *)
|
|
@@ -38,11 +37,16 @@ let error_unless_nbdkit_working () =
|
|
if not (Nbdkit.is_installed ()) then
|
|
error (f_"nbdkit is not installed or not working")
|
|
|
|
-let error_unless_nbdkit_min_version config =
|
|
+let error_unless_nbdkit_version_ge config min_version =
|
|
let version = Nbdkit.version config in
|
|
- if version < nbdkit_min_version then
|
|
- error (f_"nbdkit is too old. nbdkit >= %s is required.")
|
|
- nbdkit_min_version_string
|
|
+ if version < min_version then (
|
|
+ let min_major, min_minor, min_release = min_version in
|
|
+ error (f_"nbdkit is too old. nbdkit >= %d.%d.%d is required.")
|
|
+ min_major min_minor min_release
|
|
+ )
|
|
+
|
|
+let error_unless_nbdkit_min_version config =
|
|
+ error_unless_nbdkit_version_ge config nbdkit_min_version
|
|
|
|
let error_unless_nbdkit_plugin_exists plugin =
|
|
if not (Nbdkit.probe_plugin plugin) then
|
|
@@ -297,23 +301,35 @@ let create_ssh ?bandwidth ~password ?port ~server ?user path =
|
|
common_create ?bandwidth password "ssh" (get_args ())
|
|
|
|
(* Create an nbdkit module specialized for reading from Curl sources. *)
|
|
-let create_curl ?bandwidth ?cookie ~password ?(sslverify=true) ?user url =
|
|
+let create_curl ?bandwidth ?cookie_script ?cookie_script_renew
|
|
+ ?(sslverify=true) url =
|
|
error_unless_nbdkit_plugin_exists "curl";
|
|
|
|
+ (* The cookie* parameters require nbdkit 1.22, so check that early. *)
|
|
+ if cookie_script <> None || cookie_script_renew <> None then (
|
|
+ let config = Nbdkit.config () in
|
|
+ error_unless_nbdkit_version_ge config (1, 22, 0)
|
|
+ );
|
|
+
|
|
let add_arg, get_args =
|
|
let args = ref [] in
|
|
let add_arg (k, v) = List.push_front (k, v) args in
|
|
let get_args () = List.rev !args in
|
|
add_arg, get_args in
|
|
|
|
- Option.may (fun s -> add_arg ("user", s)) user;
|
|
(* https://bugzilla.redhat.com/show_bug.cgi?id=1146007#c10 *)
|
|
add_arg ("timeout", "2000");
|
|
- Option.may (fun s -> add_arg ("cookie", s)) cookie;
|
|
+ Option.may (fun s -> add_arg ("cookie-script", s)) cookie_script;
|
|
+ Option.may (fun i -> add_arg ("cookie-script-renew", string_of_int i))
|
|
+ cookie_script_renew;
|
|
if not sslverify then add_arg ("sslverify", "false");
|
|
add_arg ("url", url);
|
|
|
|
- common_create ?bandwidth password "curl" (get_args ())
|
|
+ (* For lots of extra debugging, uncomment one or both lines. *)
|
|
+ (*add_arg ("--debug", "curl.verbose=1");*)
|
|
+ (*add_arg ("--debug", "curl.scripts=1");*)
|
|
+
|
|
+ common_create ?bandwidth NoPassword "curl" (get_args ())
|
|
|
|
let run cmd =
|
|
let sock, _ = Nbdkit.run_unix cmd in
|
|
diff --git a/v2v/nbdkit_sources.mli b/v2v/nbdkit_sources.mli
|
|
index 94810ea6..922642df 100644
|
|
--- a/v2v/nbdkit_sources.mli
|
|
+++ b/v2v/nbdkit_sources.mli
|
|
@@ -60,10 +60,9 @@ val create_ssh : ?bandwidth:Types.bandwidth ->
|
|
Note this doesn't run nbdkit yet, it just creates the object. *)
|
|
|
|
val create_curl : ?bandwidth:Types.bandwidth ->
|
|
- ?cookie:string ->
|
|
- password:password ->
|
|
+ ?cookie_script:string ->
|
|
+ ?cookie_script_renew:int ->
|
|
?sslverify:bool ->
|
|
- ?user:string ->
|
|
string -> Nbdkit.cmd
|
|
(** Create a nbdkit object using the Curl plugin. The required
|
|
string parameter is the URL.
|
|
diff --git a/v2v/parse_libvirt_xml.ml b/v2v/parse_libvirt_xml.ml
|
|
index 0b136839..fffc5a24 100644
|
|
--- a/v2v/parse_libvirt_xml.ml
|
|
+++ b/v2v/parse_libvirt_xml.ml
|
|
@@ -319,8 +319,7 @@ let parse_libvirt_xml ?bandwidth ?conn xml =
|
|
| _, Some port ->
|
|
invalid_arg "invalid port number in libvirt XML" in
|
|
sprintf "%s://%s%s%s" driver host port (uri_quote path) in
|
|
- let nbdkit = Nbdkit_sources.create_curl ?bandwidth ~password:NoPassword
|
|
- url in
|
|
+ let nbdkit = Nbdkit_sources.create_curl ?bandwidth url in
|
|
let qemu_uri = Nbdkit_sources.run nbdkit in
|
|
add_disk qemu_uri format controller P_dont_rewrite
|
|
| Some protocol, _, _ ->
|
|
diff --git a/v2v/vCenter.ml b/v2v/vCenter.ml
|
|
index 4c128b0c..ead03364 100644
|
|
--- a/v2v/vCenter.ml
|
|
+++ b/v2v/vCenter.ml
|
|
@@ -46,11 +46,12 @@ let rec map_source ?bandwidth ?password_file dcPath uri server path =
|
|
(* XXX only works if the query string is not URI-quoted *)
|
|
String.find query "no_verify=1" = -1 in
|
|
|
|
+ (* Check the URL exists and authentication info is correct. *)
|
|
let https_url =
|
|
let https_url = get_https_url dcPath uri server path in
|
|
- (* Check the URL exists. *)
|
|
- let status, _, _ =
|
|
+ let status, _, dump_response =
|
|
fetch_headers_from_url password_file uri sslverify https_url in
|
|
+
|
|
(* If a disk is actually a snapshot image it will have '-00000n'
|
|
* appended to its name, e.g.:
|
|
* [yellow:storage1] RHEL4-X/RHEL4-X-000003.vmdk
|
|
@@ -58,28 +59,71 @@ let rec map_source ?bandwidth ?password_file dcPath uri server path =
|
|
* a 404 and the vmdk name looks like it might be a snapshot, try
|
|
* again without the snapshot suffix.
|
|
*)
|
|
- if status = "404" && PCRE.matches snapshot_re path then (
|
|
- let path = PCRE.sub 1 ^ PCRE.sub 2 in
|
|
- get_https_url dcPath uri server path
|
|
- )
|
|
- else
|
|
- (* Note that other non-200 status errors will be handled
|
|
- * in get_session_cookie below, so we don't have to worry
|
|
- * about them here.
|
|
- *)
|
|
- https_url in
|
|
+ let https_url, status, dump_response =
|
|
+ if status = "404" && PCRE.matches snapshot_re path then (
|
|
+ let path = PCRE.sub 1 ^ PCRE.sub 2 in
|
|
+ let https_url = get_https_url dcPath uri server path in
|
|
+ let status, _, dump_response =
|
|
+ fetch_headers_from_url password_file uri sslverify https_url in
|
|
+ https_url, status, dump_response
|
|
+ )
|
|
+ else (https_url, status, dump_response) in
|
|
+
|
|
+ if status = "401" then (
|
|
+ dump_response stderr;
|
|
+ if uri.uri_user <> None then
|
|
+ error (f_"vcenter: incorrect username or password")
|
|
+ else
|
|
+ error (f_"vcenter: incorrect username or password. You might need to specify the username in the URI like this: [vpx|esx|..]://USERNAME@[etc]")
|
|
+ );
|
|
+
|
|
+ if status = "404" then (
|
|
+ dump_response stderr;
|
|
+ error (f_"vcenter: URL not found: %s") https_url
|
|
+ );
|
|
+
|
|
+ if status <> "200" then (
|
|
+ dump_response stderr;
|
|
+ error (f_"vcenter: invalid response from server: %s") status
|
|
+ );
|
|
+
|
|
+ https_url in
|
|
|
|
let session_cookie =
|
|
get_session_cookie password_file uri sslverify https_url in
|
|
|
|
- let password =
|
|
- match password_file with
|
|
- | None -> Nbdkit_sources.NoPassword
|
|
- | Some password_file -> Nbdkit_sources.PasswordFile password_file in
|
|
+ (* Write a cookie script to retrieve the session cookie.
|
|
+ * See nbdkit-curl-plugin(1) "Example: VMware ESXi cookies"
|
|
+ *)
|
|
+ let cookie_script, chan =
|
|
+ Filename.open_temp_file ~perms:0o700 "v2vcs" ".sh" in
|
|
+ unlink_on_exit cookie_script;
|
|
+ let fpf fs = fprintf chan fs in
|
|
+ fpf "#!/bin/sh -\n";
|
|
+ fpf "\n";
|
|
+ fpf "curl --head -s";
|
|
+ if not sslverify then fpf " --insecure";
|
|
+ (match uri.uri_user, password_file with
|
|
+ | None, None -> ()
|
|
+ | Some user, None -> fpf " -u %s" (quote user)
|
|
+ | None, Some password_file ->
|
|
+ fpf " -u \"$LOGNAME\":\"$(cat %s)\"" (quote password_file)
|
|
+ | Some user, Some password_file ->
|
|
+ fpf " -u %s:\"$(cat %s)\"" (quote user) (quote password_file)
|
|
+ );
|
|
+ fpf " %s" (quote https_url);
|
|
+ fpf " |\n";
|
|
+ fpf "\tsed -ne %s\n" (quote "{ s/^Set-Cookie: \\([^;]*\\);.*/\\1/ip }");
|
|
+ close_out chan;
|
|
+
|
|
+ (* VMware authentication expires after 30 minutes so we must renew
|
|
+ * after < 30 minutes.
|
|
+ *)
|
|
+ let cookie_script_renew = 25*60 in
|
|
|
|
let nbdkit =
|
|
- Nbdkit_sources.create_curl ?bandwidth ?cookie:session_cookie ~password ~sslverify
|
|
- ?user:uri.uri_user https_url in
|
|
+ Nbdkit_sources.create_curl ?bandwidth ~cookie_script ~cookie_script_renew
|
|
+ ~sslverify https_url in
|
|
let qemu_uri = Nbdkit_sources.run nbdkit in
|
|
|
|
(* Return the struct. *)
|