xend: Add support for URI ('file:' and 'data:' scheme) for PV/kernel

and PV/ramdisk

Add support for 'file:' and 'data:' URI schemes for the parameters
'PV/kernel' and 'PV/ramdisk' in the VM.create() call. The 'data:'
scheme handling enables using a file which is stored inside the
management system (from where the XenAPI call is send) as kernel or
ramdisk.

Notes:
o all included: a detailed description can be found in the xenapi
documentation
o bumped up the version of the API document to 1.0.8 (because of
(minimal) interface extension)
o Future enhancements (like http:, ftp: schemes) fit seamlessly into
the current design / classes
o Unittest cases and xm-test case included

Signed-off-by: Andreas Florath <xen@flonatel.org>

5 files changed, 566 insertions, 0 deletions

diff --git a/tools/tests/run_tests.sh b/tools/tests/run_tests.sh
new file mode 100644
index 0000000000..c492876b4b
--- /dev/null
+++ b/tools/tests/run_tests.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# This runs the available unit-tests with all different supported
+# python versions.
+# 
+# To run this this must be 'cd'ed to the tests directory.
+#
+
+ENABLE_UNSUPPORTED=0
+
+function usage()
+{
+    printf "Usage: %s: [-u]\n" $0
+    printf "   -u: run test with unsupported python versions also\n"
+}
+
+function run_one_test()
+{
+    PYTHON=$1
+    PYTHON_EXECUTABLE=`echo $PYTHON | tr -d "-"`
+    echo "+++ Running tests with $PYTHON"
+    export LD_LIBRARY_PATH=./regression/installed/$PYTHON/lib
+    ./regression/installed/$PYTHON/bin/$PYTHON_EXECUTABLE \
+	utests/run_all_tests.py
+    echo "--- Finished tests with $PYTHON"
+}
+
+function run_all_tests()
+{
+    for PYTHON in $@;
+    do
+	run_one_test $PYTHON
+    done
+}
+
+while getopts u name
+do
+    case $name in
+	h)  usage; exit 0;;
+	u)  ENABLE_UNSUPPORTED=1;;
+	?)  usage; exit 2;;
+    esac
+done
+
+# Build the different python versions
+(cd regression && make -j4 runtime-environment)
+
+# Supported: when an unit test fails this should be seen as an error
+PYTHON_SUPPORTED="python-2.4 python-2.5 python-2.6"
+# Unsupported: failure should be seen as a hint
+PYTHON_UNSUPPORTED="python-3.1"
+
+export PYTHONPATH=`echo $PWD/../python/build/lib.*`:$PWD
+
+set -e
+run_all_tests $PYTHON_SUPPORTED
+
+if test $ENABLE_UNSUPPORTED -eq 1
+then
+    run_all_tests $PYTHON_UNSUPPORTED
+fi
diff --git a/tools/tests/utests/run_all_tests.py b/tools/tests/utests/run_all_tests.py
new file mode 100644
index 0000000000..e36fa4d836
--- /dev/null
+++ b/tools/tests/utests/run_all_tests.py
@@ -0,0 +1,32 @@
+#============================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2009 flonatel GmbH & Co. KG
+#============================================================================
+
+import unittest
+
+import utests.ut_util.ut_fileuri
+import utests.ut_xend.ut_XendConfig
+import utests.ut_xend.ut_image
+
+suite = unittest.TestSuite(
+    [utests.ut_util.ut_fileuri.suite(),
+     utests.ut_xend.ut_XendConfig.suite(),
+     utests.ut_xend.ut_image.suite(),
+     ])
+
+if __name__ == "__main__":
+    testresult = unittest.TextTestRunner(verbosity=3).run(suite)
+
diff --git a/tools/tests/utests/ut_util/ut_fileuri.py b/tools/tests/utests/ut_util/ut_fileuri.py
new file mode 100644
index 0000000000..cd64e7d05e
--- /dev/null
+++ b/tools/tests/utests/ut_util/ut_fileuri.py
@@ -0,0 +1,209 @@
+#============================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2009 flonatel GmbH & Co. KG
+#============================================================================
+
+import os
+import unittest
+
+from xen.util.fileuri import scheme_error
+from xen.util.fileuri import scheme_data
+from xen.util.fileuri import scheme_file
+from xen.util.fileuri import schemes
+
+class scheme_data_unit_tests(unittest.TestCase):
+
+    def check_basic_encoding(self):
+        "util.fileuri.scheme_data - basic encoding"
+        sd = scheme_data.encode('Hello!')
+        self.assertEqual(sd, 'data:application/octet-stream;base64,SGVsbG8h')
+
+    def check_encoding_with_given_mediatype(self):
+        "util.fileuri.scheme_data - encoding with given media name"
+        sd = scheme_data.encode('Hello!', 'application/x-my-linux-kernel')
+        self.assertEqual(sd,
+              'data:application/x-my-linux-kernel;base64,SGVsbG8h')
+
+    def check_parse_01(self):
+        "util.fileuri.scheme_data - parsing of None"
+        self.assertRaises(scheme_error, scheme_data.parse, None)
+
+    def check_parse_02(self):
+        "util.fileuri.scheme_data - parsing of empty string"
+        self.assertRaises(scheme_error, scheme_data.parse, "")
+
+    def check_parse_03(self):
+        "util.fileuri.scheme_data - parsing of unstructured data"
+        self.assertRaises(scheme_error, scheme_data.parse, "akskdjdfhezezu")
+
+    def check_parse_04(self):
+        "util.fileuri.scheme_data - data: is not at the first place"
+        self.assertRaises(scheme_error, scheme_data.parse, 'ggdata:sossm')
+
+    def check_parse_05(self):
+        "util.fileuri.scheme_data - no comma in data"
+        self.assertRaises(scheme_error, scheme_data.parse, 'data:sossm')
+
+    def check_parse_06(self):
+        "util.fileuri.scheme_data - encoding is empty"
+        self.assertRaises(scheme_error, scheme_data.parse, 'data:,')
+
+    def check_parse_07(self):
+        "util.fileuri.scheme_data - unknown encoding"
+        self.assertRaises(scheme_error, scheme_data.parse,
+                          'data:somemediatype;unknown,')
+
+    def check_parse_08(self):
+        "util.fileuri.scheme_data - parse ok - empty data"
+        mediatype, encoding, data_start = scheme_data.parse(
+            'data:somemedia;base64,')
+        self.assertEqual(mediatype, 'somemedia')
+        self.assertEqual(encoding, 'base64')
+        self.assertEqual(data_start, 22)
+
+    def check_parse_09(self):
+        "util.fileuri.scheme_data - parse ok - some data"
+        mediatype, encoding, data_start = scheme_data.parse(
+            'data:somemedia;base64,HereComesTheSun')
+        self.assertEqual(mediatype, 'somemedia')
+        self.assertEqual(encoding, 'base64')
+        self.assertEqual(data_start, 22)
+
+    def check_parse_10(self):
+        "util.fileuri.scheme_data - header ok - data error"
+        self.assertRaises(scheme_error, scheme_data.decode,
+               'data:application/octet-stream;base64,H!$ere"Co<mesT>heS_.un')
+
+    def check_cff_file_does_not_exist(self):
+        "util.fileuri.scheme_data - create from file - non existent file"
+        self.assertRaises(scheme_error, scheme_data.create_from_file,
+                          "/there/is/hopefully/no/file/like/this")
+
+    def check_cff_ok(self):
+        "util.fileuri.scheme_data - create from file - ok"
+        tmppath = "/tmp/scheme_data_check_cff_ok"
+        f = open(tmppath, "w")
+        f.write("huhuhu")
+        f.close()
+        d = scheme_data.create_from_file(tmppath)
+        os.unlink(tmppath)
+        self.assertEqual(d, "data:application/octet-stream;base64,aHVodWh1")
+
+
+class scheme_file_unit_tests(unittest.TestCase):
+
+    def check_encode_empty_filename(self):
+        "util.fileuri.scheme_file - encode empty filename"
+        self.assertRaises(scheme_error, scheme_file.encode, "")
+
+    def check_encode_relative_filename(self):
+        "util.fileuri.scheme_file - encode relative filename"
+        self.assertRaises(scheme_error, scheme_file.encode, "../there")
+
+    def check_encode_absolut_filename(self):
+        "util.fileuri.scheme_file - encode absolut filename"
+        self.assertEqual(
+            scheme_file.encode("/here/and/there/again"),
+            'file:///here/and/there/again')
+
+    def check_decode_01(self):
+        "util.fileuri.scheme_file - decode empty data"
+        self.assertRaises(scheme_error, scheme_file.decode, "")
+
+    def check_decode_02(self):
+        "util.fileuri.scheme_file - decode data with no file:// at the beginning (1)"
+        self.assertRaises(scheme_error, scheme_file.decode,
+                          "phonehome://bbbb")
+
+    def check_decode_03(self):
+        "util.fileuri.scheme_file - decode data with no file:// at the beginning (2)"
+        self.assertRaises(scheme_error, scheme_file.decode,
+                          "file:/bbbb")
+
+    def check_decode_04(self):
+        "util.fileuri.scheme_file - decode empty path"
+        self.assertRaises(scheme_error, scheme_file.decode,
+                          "file://")
+
+    def check_decode_05(self):
+        "util.fileuri.scheme_file - decode empty relative path"
+        self.assertRaises(scheme_error, scheme_file.decode,
+                          "file://somewhere")
+
+    def check_decode_06(self):
+        "util.fileuri.scheme_file - decode ok"
+        path, tmp_file = scheme_file.decode("file:///boot/vmlinuz")
+        self.assertEqual(path, "/boot/vmlinuz")
+        self.assertEqual(tmp_file, False)
+
+class scheme_set_unit_tests(unittest.TestCase):
+
+    def check_data_01(self):
+        "util.fileuri.scheme_set - data with error in media type"
+
+        u = "data:something_wrong,base64:swer"
+        uri, tmp_file = schemes.decode(u)
+        self.assertEqual(uri, u)
+        self.assertEqual(tmp_file, False)
+
+    def check_data_02(self):
+        "util.fileuri.scheme_set - data with error in base64 data"
+
+        u = "data:application/octet-stream;base64,S!VsbG8h"
+        uri, tmp_file = schemes.decode(u)
+        self.assertEqual(uri, u)
+        self.assertEqual(tmp_file, False)
+ 
+    def check_data_03(self):
+        "util.fileuri.scheme_set - data ok"
+
+        u = "data:application/octet-stream;base64,SGVsbG8h"
+        uri, tmp_file = schemes.decode(u)
+
+        # Read file contents
+        f = open(uri, "r")
+        d = f.read()
+        f.close()
+        os.unlink(uri)
+
+        self.assertEqual(d, "Hello!")
+        self.assertEqual(tmp_file, True)
+       
+    def check_file_01(self):
+        "util.fileuri.scheme_set - file ok"
+
+        f = "/The/Path/To/The/File.txt"
+        uri, tmp_file = schemes.decode("file://" + f)
+        self.assertEqual(uri, f)
+        self.assertEqual(tmp_file, False)
+
+    def check_without_scheme_01(self):
+        "util.fileuri.scheme_set - without scheme"
+
+        f = "/The/Path/To/The/File.txt"
+        uri, tmp_file = schemes.decode(f)
+        self.assertEqual(uri, f)
+        self.assertEqual(tmp_file, False)
+
+
+def suite():
+    return unittest.TestSuite(
+        [unittest.makeSuite(scheme_data_unit_tests, 'check_'),
+         unittest.makeSuite(scheme_file_unit_tests, 'check_'),
+         unittest.makeSuite(scheme_set_unit_tests, 'check_'),])
+
+if __name__ == "__main__":
+    testresult = unittest.TextTestRunner(verbosity=3).run(suite())
+
diff --git a/tools/tests/utests/ut_xend/ut_XendConfig.py b/tools/tests/utests/ut_xend/ut_XendConfig.py
new file mode 100644
index 0000000000..724ad084b3
--- /dev/null
+++ b/tools/tests/utests/ut_xend/ut_XendConfig.py
@@ -0,0 +1,117 @@
+#===========================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2009 flonatel GmbH & Co. KG
+#============================================================================
+
+import os
+import unittest
+
+# This does not work because of a cyclic import loop
+#from xen.xend.XendConfig import XendConfig
+import xen.xend.XendDomain
+
+class XendConfigUnitTest(unittest.TestCase):
+
+    def minimal_vmconf(self):
+        return {
+            'memory_dynamic_min': 64,
+            'memory_dynamic_max': 128,
+            'memory_static_max': 128,
+            }
+
+    def check_hf_01(self):
+        "xend.XendConfig.handle_fileutils - PV_kernel/ramdisk not set"
+        vmconf = self.minimal_vmconf()
+        xc = xen.xend.XendConfig.XendConfig(xapi = vmconf)
+
+        self.assert_(not xc.has_key('use_tmp_kernel'))
+        self.assert_(not xc.has_key('use_tmp_ramdisk'))
+
+    def check_hf_02(self):
+        "xend.XendConfig.handle_fileutils - PV_kernel/ramdisk set to some path"
+        vmconf = self.minimal_vmconf()
+        vmconf['PV_kernel'] = '/some/where/under/the/rainbow-kernel'
+        vmconf['PV_ramdisk'] = '/some/where/under/the/rainbow-ramdisk'
+        xc = xen.xend.XendConfig.XendConfig(xapi = vmconf)
+
+        self.assert_(xc.has_key('use_tmp_kernel'))
+        self.assert_(xc.has_key('use_tmp_ramdisk'))
+
+        self.assert_(not xc['use_tmp_kernel'])
+        self.assert_(not xc['use_tmp_ramdisk'])
+
+    def check_hf_03(self):
+        "xend.XendConfig.handle_fileutils - PV_kernel/ramdisk using file: scheme"
+        vmconf = self.minimal_vmconf()
+        vmconf['PV_kernel'] = 'file:///some/where/under/the/rainbow-kernel'
+        vmconf['PV_ramdisk'] = 'file:///some/where/under/the/rainbow-ramdisk'
+        xc = xen.xend.XendConfig.XendConfig(xapi = vmconf)
+
+        self.assert_(xc.has_key('use_tmp_kernel'))
+        self.assert_(xc.has_key('use_tmp_ramdisk'))
+
+        self.assert_(not xc['use_tmp_kernel'])
+        self.assert_(not xc['use_tmp_ramdisk'])
+
+        self.assert_('PV_kernel' in xc)
+        self.assert_('PV_ramdisk' in xc)
+
+        self.assertEqual("/some/where/under/the/rainbow-kernel",
+                         xc['PV_kernel'])
+        self.assertEqual("/some/where/under/the/rainbow-ramdisk",
+                         xc['PV_ramdisk'])
+
+    def check_hf_04(self):
+        "xend.XendConfig.handle_fileutils - PV_kernel/ramdisk using data: scheme"
+        vmconf = self.minimal_vmconf()
+        vmconf['PV_kernel'] = 'data:application/octet-stream;base64,VGhpcyBpcyB0aGUga2VybmVsCg=='
+        vmconf['PV_ramdisk'] = 'data:application/octet-stream;base64,TXkgZ3JlYXQgcmFtZGlzawo='
+        xc = xen.xend.XendConfig.XendConfig(xapi = vmconf)
+
+        self.assert_(xc.has_key('use_tmp_kernel'))
+        self.assert_(xc.has_key('use_tmp_ramdisk'))
+
+        self.assert_(xc['use_tmp_kernel'])
+        self.assert_(xc['use_tmp_ramdisk'])
+
+        self.assert_('PV_kernel' in xc)
+        self.assert_('PV_ramdisk' in xc)
+
+        self.assert_(xc['PV_kernel'].startswith(
+                "/var/run/xend/boot/data_uri_file."))
+        self.assert_(xc['PV_ramdisk'].startswith(
+                "/var/run/xend/boot/data_uri_file."))
+
+        f = file(xc['PV_kernel'])
+        kc = f.read()
+        f.close()
+
+        f = file(xc['PV_ramdisk'])
+        rc = f.read()
+        f.close()
+
+        os.unlink(xc['PV_kernel'])
+        os.unlink(xc['PV_ramdisk'])
+
+        self.assertEqual(kc, "This is the kernel\n")
+        self.assertEqual(rc, "My great ramdisk\n")
+
+def suite():
+    return unittest.TestSuite(
+        [unittest.makeSuite(XendConfigUnitTest, 'check_'),])
+
+if __name__ == "__main__":
+    testresult = unittest.TextTestRunner(verbosity=3).run(suite())
+
diff --git a/tools/tests/utests/ut_xend/ut_image.py b/tools/tests/utests/ut_xend/ut_image.py
new file mode 100644
index 0000000000..92ec6458f7
--- /dev/null
+++ b/tools/tests/utests/ut_xend/ut_image.py
@@ -0,0 +1,147 @@
+#===========================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2009 flonatel GmbH & Co. KG
+#============================================================================
+
+import unittest
+import tempfile
+import os
+
+import xen.xend.image
+
+class ImageHandlerUnitTests(unittest.TestCase):
+
+    class ImageHandlerUnitTestsVirtualMachine:
+
+        def __init__(self):
+            self.info = {
+                'name_label': 'ItsMyParty',
+                }
+
+        def storeVm(self, *args):
+            pass
+
+        def permissionsVm(self, *args):
+            pass
+
+        def getDomid(self):
+            return 7
+
+    # Sets up a vm_config with no bootloader.
+    def vm_config_no_bootloader(self):
+        return {
+            'PV_kernel': 'value_of_PV_kernel',
+            'PV_args': 'value_of_PV_args',
+            'PV_ramdisk': 'value_of_PV_ramdisk',
+            'platform': {},
+            'console_refs': [],
+            }
+
+    def check_configure_01(self):
+        # This retests the problem reported by Jun Koi on 24.07.2009
+        # see http://lists.xensource.com/archives/html/xen-devel/2009-07/msg01006.html
+        "ImageHandler - call configure with mostly empty vmConfig"
+
+        vmConfig = self.vm_config_no_bootloader()
+        vm = self.ImageHandlerUnitTestsVirtualMachine()
+        ih = xen.xend.image.ImageHandler(vm, vmConfig)
+
+        self.assertEqual(ih.use_tmp_kernel, False)
+        self.assertEqual(ih.use_tmp_ramdisk, False)
+
+    def check_configure_02(self):
+        "ImageHandler - call configure with use_tmp_xxx set to false"
+
+        vmConfig = self.vm_config_no_bootloader()
+        vmConfig['use_tmp_kernel'] = False
+        vmConfig['use_tmp_ramdisk'] = False
+        vm = self.ImageHandlerUnitTestsVirtualMachine()
+        ih = xen.xend.image.ImageHandler(vm, vmConfig)
+
+        self.assertEqual(ih.use_tmp_kernel, False)
+        self.assertEqual(ih.use_tmp_ramdisk, False)
+
+
+    def check_configure_03(self):
+        "ImageHandler - call configure with use_tmp_xxx set to true"
+
+        vmConfig = self.vm_config_no_bootloader()
+        vmConfig['use_tmp_kernel'] = True
+        vmConfig['use_tmp_ramdisk'] = True
+        vm = self.ImageHandlerUnitTestsVirtualMachine()
+        ih = xen.xend.image.ImageHandler(vm, vmConfig)
+
+        self.assertEqual(ih.use_tmp_kernel, True)
+        self.assertEqual(ih.use_tmp_ramdisk, True)
+
+    def cleanup_tmp_images_base(self, vmConfig):
+        vm = self.ImageHandlerUnitTestsVirtualMachine()
+        ih = xen.xend.image.ImageHandler(vm, vmConfig)
+
+        k, ih.kernel = tempfile.mkstemp(
+            prefix = "ImageHandler-cleanupTmpImages-k", dir = "/tmp")
+        r, ih.ramdisk = tempfile.mkstemp(
+            prefix = "ImageHandler-cleanupTmpImages-r", dir = "/tmp")
+
+        ih.cleanupTmpImages()
+
+        kres = os.path.exists(ih.kernel)
+        rres = os.path.exists(ih.ramdisk)
+
+        if not ih.use_tmp_kernel:
+            os.unlink(ih.kernel)
+        if not ih.use_tmp_ramdisk:
+            os.unlink(ih.ramdisk)
+
+        return kres, rres
+
+    def check_cleanup_tmp_images_01(self):
+        "ImageHandler - cleanupTmpImages with use_tmp_xxx unset"
+
+        vmConfig = self.vm_config_no_bootloader()
+        kres, rres = self.cleanup_tmp_images_base(vmConfig)
+
+        self.assertEqual(kres, True)
+        self.assertEqual(rres, True)
+
+    def check_cleanup_tmp_images_02(self):
+        "ImageHandler - cleanupTmpImages with use_tmp_xxx set to false"
+
+        vmConfig = self.vm_config_no_bootloader()
+        vmConfig['use_tmp_kernel'] = False
+        vmConfig['use_tmp_ramdisk'] = False
+        kres, rres = self.cleanup_tmp_images_base(vmConfig)
+
+        self.assertEqual(kres, True)
+        self.assertEqual(rres, True)
+
+    def check_cleanup_tmp_images_03(self):
+        "ImageHandler - cleanupTmpImages with use_tmp_xxx set to true"
+
+        vmConfig = self.vm_config_no_bootloader()
+        vmConfig['use_tmp_kernel'] = True
+        vmConfig['use_tmp_ramdisk'] = True
+        kres, rres = self.cleanup_tmp_images_base(vmConfig)
+
+        self.assertEqual(kres, False)
+        self.assertEqual(rres, False)
+
+def suite():
+    return unittest.TestSuite(
+        [unittest.makeSuite(ImageHandlerUnitTests, 'check_'),])
+
+if __name__ == "__main__":
+    testresult = unittest.TextTestRunner(verbosity=3).run(suite())
+