Changeset 103028 in vbox for trunk/src/VBox/Installer/common

Timestamp:

Jan 24, 2024 3:53:59 PM (13 months ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

161263

Message:

Main/Python: Big revamp to modernize our vboxapi Python package to now use a so-called src-layout, which also can be used with pip directly. Also added compatibility w/ Python 3.12 where distutils are not shipped anymore. We also now do have automatic linting for our code, which hopefully should improve quality in this area. Moved some Python-related files into an own sub folder so that it's more clear to which these belong to. The "sdk/installer" directories also have an own "python" sub directory where the stuff resides now. The Windows installer also uses "sdk/installer" instead of "sdk/install", to match the other platforms. bugref:10579

Location:

trunk/src/VBox/Installer/common

Files:

• Config.kmk (copied) (copied from trunk/src/VBox/Frontends/VBoxShell/Config.kmk ) (5 diffs)
• Makefile.kmk (modified) (1 diff)
• vboxapisetup-stub.py (added)
• vboxapisetup.py (modified) (1 diff)

trunk/src/VBox/Installer/common/Config.kmk

@@ -1 +1 @@
 # $Id$
 ## @file
-# kBuild Configuration file for VBoxShell.
+# kBuild Configuration file for common installer parts.
 #
 
@@ -26 +26 @@
 #
 
-VBOX_VBOXSHELL_CONFIG_KMK_INCLUDED = 1
+VBOX_INSTALLER_COMMON_CONFIG_KMK_INCLUDED = 1
 
 # Include the top-level configure file.
@@ -36 +36 @@
 # Globals
 #
-VBOX_PATH_VBOXSHELL_SRC := $(PATH_ROOT)/src/VBox/Frontends/VBoxShell
+VBOX_PATH_INSTALLER_COMMON_SRC := $(PATH_ROOT)/src/VBox/Installer/common
 
 #
 # List of python sources that should be linted.
 #
-VBOX_VBOXSHELL_PYTHON_SOURCES     :=
-VBOX_VBOXSHELL_PYLINT_TARGETS     :=
+VBOX_INSTALLER_COMMON_PYTHON_SOURCES :=
+VBOX_INSTALLER_COMMON_PYLINT_TARGETS :=
 
 ifdef VBOX_WITH_PYLINT
@@ -51 +51 @@
 # Process python sources.
 #
-if1of ($(KBUILD_TARGET), win os2)
- VBOX_PYTHONPATH_VBOXSHELL = $(PYTHONPATH);$(VBOX_PATH_VBOXSHELL_SRC)
-else
- VBOX_PYTHONPATH_VBOXSHELL = $(PYTHONPATH):$(VBOX_PATH_VBOXSHELL_SRC)
-endif
+VBOX_PYTHONPATH_INSTALLER_COMMON = $(PYTHONPATH):$(VBOX_PATH_INSTALLER_COMMON_SRC)
 BLDDIRS += $(PATH_TARGET)/pylint
 
-define def_vbox_vboxshell_py_check
+define def_vbox_installer_common_py_check
  $(eval name:=$(basename $(notdir $(py))))
 
@@ -66 +62 @@
  ifdef VBOX_WITH_PYLINT
         $(QUIET2)$(call MSG_L1,Subjecting $(py) to pylint...)
-        $(QUIET)$(REDIRECT) -C "$(dir $(py))" -E LC_ALL=C -E PYTHONPATH="$(VBOX_PYTHONPATH_VBOXSHELL)" -- \
-                $(VBOX_PYLINT) --rcfile=$(VBOX_PATH_VBOXSHELL_SRC)/pylintrc $$(VBOX_PYLINT_FLAGS) $$($(py)_VBOX_PYLINT_FLAGS) ./$(notdir $(py))
+        $(QUIET)$(REDIRECT) -C "$(dir $(py))" -E LC_ALL=C -E PYTHONPATH="$(VBOX_PYTHONPATH_INSTALLER_COMMON)" -- \
+                $(VBOX_PYLINT) --rcfile=$(VBOX_PATH_INSTALLER_COMMON_SRC)/pylintrc $$(VBOX_PYLINT_FLAGS) $$($(py)_VBOX_PYLINT_FLAGS) ./$(notdir $(py))
  endif
         $(QUIET)$(APPEND) -t "$(PATH_TARGET)/pylint/$(name).o"
  TESTING += $(name)-py-phony.o
- VBOX_VBOXSHELL_PYLINT_TARGETS    += $(PATH_TARGET)/pylint/$(name).o
-endef # def_vbox_vboxshell_py_check
+ VBOX_INSTALLER_COMMON_PYLINT_TARGETS    += $(PATH_TARGET)/pylint/$(name).o
+endef # def_vbox_installer_common_py_check
 
 
-define def_vbox_vboxshell_process_python_sources
+define def_vbox_installer_common_process_python_sources
  if $(words $(_SUB_MAKEFILE_STACK)) <= 0 || "$1" == "FORCE"
-  $(foreach py, $(VBOX_VBOXSHELL_PYTHON_SOURCES), $(eval $(def_vbox_vboxshell_py_check)))
+  $(foreach py, $(VBOX_INSTALLER_COMMON_PYTHON_SOURCES), $(eval $(def_vbox_installer_common_py_check)))
  endif
 endef

trunk/src/VBox/Installer/common/Makefile.kmk

@@ -37 +37 @@
 ifdef VBOX_WITH_PYTHON
 
- INSTALLS += VBox-python-glue-installer
+ INSTALLS += VBox-python-glue-vboxapisetup
 
- VBox-python-glue-installer_INST = $(INST_SDK)installer/
- VBox-python-glue-installer_MODE = a+r,u+w
- VBox-python-glue-installer_SOURCES = vboxapisetup.py
+ VBox-python-glue-vboxapisetup_INST = $(INST_SDK)installer/python/vboxapi/
+ VBox-python-glue-vboxapisetup_MODE = a+r,u+w
+ VBox-python-glue-vboxapisetup_SOURCES = vboxapisetup.py=>setup.py
+
+ INSTALLS += VBox-python-glue-vboxapisetup-stub
+
+ VBox-python-glue-vboxapisetup-stub_INST = $(INST_SDK)installer/python/
+ VBox-python-glue-vboxapisetup-stub_MODE = a+r,u+w
+ VBox-python-glue-vboxapisetup-stub_SOURCES = vboxapisetup-stub.py=>vboxapisetup.py
+
+ #
+ # Automatically lint common installer Python stuff.
+ #
+ if defined(VBOX_WITH_PYLINT) && !defined(VBOX_WITHOUT_AUTO_PYLINT)
+  OTHERS      += $(PATH_TARGET)/pylintInstallerCommon.run
+  OTHER_CLEAN += $(PATH_TARGET)/pylintInstallerCommon.run
+  $(PATH_TARGET)/pylintInstallerCommon.run: \
+        ${PATH_SUB_CURRENT}/vboxapisetup.py ${PATH_SUB_CURRENT}/vboxapisetup-stub.py
+        $(QUIET)$(APPEND) -t "$@"
+ endif
+
+ VBOX_INSTALLER_COMMON_PYTHON_SOURCES := $(wildcard $(PATH_SUB_CURRENT)/*.py)
+
+ $(evalcall def_vbox_installer_common_process_python_sources,FORCE)
 
 endif # VBOX_WITH_PYTHON
 
 include $(FILE_KBUILD_SUB_FOOTER)
-

trunk/src/VBox/Installer/common/vboxapisetup.py

@@ -30 +30 @@
 """
 
-import os,sys
-from distutils.core import setup
-
-def cleanupComCache():
+#
+# For "modern" Python packages setuptools only is one of many ways
+# for installing a package on a system and acts as a pure build backend now.
+# Using a build backend is controlled by the accompanied pyproject.toml file.
+#
+# See: https://packaging.python.org/en/latest/discussions/setup-py-deprecated/#setup-py-deprecated
+#
+# PEP 518 [1] introduced pyproject.toml.
+# PEP 632 [2], starting with Python 3.10, the distutils module is marked as being deprecated.
+# Python 3.12 does not ship with distutils anymore, but we have to still support Python < 3.12.
+#
+# [1] https://peps.python.org/pep-0518/
+# [2] https://peps.python.org/pep-0632/
+
+import atexit
+import io
+import os
+import platform
+import sys
+
+g_fVerbose = True
+
+def cleanupWinComCacheDir(sPath):
+    """
+    Cleans up a Windows COM cache directory by deleting it.
+    """
+    if not sPath:
+        return
     import shutil
-    from distutils.sysconfig import get_python_lib
-    comCache1 = os.path.join(get_python_lib(), 'win32com', 'gen_py')
-    comCache2 = os.path.join(os.environ.get("TEMP", "c:\\tmp"), 'gen_py')
-    print("Cleaning COM cache at",comCache1,"and",comCache2)
-    shutil.rmtree(comCache1, True)
-    shutil.rmtree(comCache2, True)
-
-def patchWith(file,install,sdk):
-    newFile=file + ".new"
-    install=install.replace("\\", "\\\\")
-    try:
-        os.remove(newFile)
-    except:
+    sDirCache = os.path.join(sPath, 'win32com', 'gen_py')
+    print("Cleaning COM cache at '%s'" % (sDirCache))
+    shutil.rmtree(sDirCache, True)
+
+def cleanupWinComCache():
+    """
+    Cleans up various Windows COM cache directories by deleting them.
+    """
+    if sys.version_info >= (3, 2): # Since Python 3.2 we use the site module.
+        import site
+        for sSiteDir in site.getsitepackages():
+            cleanupWinComCacheDir(sSiteDir)
+    else:
+        from distutils.sysconfig import get_python_lib # pylint: disable=deprecated-module
+        cleanupWinComCacheDir(get_python_lib())
+
+    # @todo r=andy Do still need/want this? Was there forever. Probably a leftover.
+    sDirTemp = os.path.join(os.environ.get("TEMP", "c:\\tmp"), 'gen_py')
+    cleanupWinComCacheDir(sDirTemp)
+
+def patchWith(sFile, sVBoxInstallPath, sVBoxSdkPath):
+    """
+    Patches a given file with the VirtualBox install path + SDK path.
+    """
+    sFileTemp = sFile + ".new"
+    sVBoxInstallPath = sVBoxInstallPath.replace("\\", "\\\\")
+    try:
+        os.remove(sFileTemp)
+    except Exception as _:
         pass
-    oldF = open(file, 'r')
-    newF = open(newFile, 'w')
-    for line in oldF:
-        line = line.replace("%VBOX_INSTALL_PATH%", install)
-        line = line.replace("%VBOX_SDK_PATH%", sdk)
-        newF.write(line)
-    newF.close()
-    oldF.close()
-    try:
-        os.remove(file)
-    except:
+    # Note: We need io.open() to specify the file encoding on Python <= 2.7.
+    try:
+        with io.open(sFile, 'r', encoding='utf-8') as fileSrc:
+            with io.open(sFileTemp, 'w', encoding='utf-8') as fileDst:
+                for line in fileSrc:
+                    line = line.replace("%VBOX_INSTALL_PATH%", sVBoxInstallPath)
+                    line = line.replace("%VBOX_SDK_PATH%", sVBoxSdkPath)
+                    fileDst.write(line)
+                fileDst.close()
+            fileSrc.close()
+    except IOError as exc:
+        print("ERROR: Opening VirtualBox Python source file '%s' failed: %s" % (sFile, exc))
+        return False
+    try:
+        os.remove(sFile)
+    except Exception as _:
         pass
-    os.rename(newFile, file)
-
-# See http://docs.python.org/distutils/index.html
-def main(argv):
-    vboxDest = os.environ.get("VBOX_MSI_INSTALL_PATH", None)
-    if vboxDest is None:
-        vboxDest = os.environ.get('VBOX_INSTALL_PATH', None)
-        if vboxDest is None:
-            raise Exception("No VBOX_INSTALL_PATH defined, exiting")
-
-    vboxVersion = os.environ.get("VBOX_VERSION", None)
-    if vboxVersion is None:
+    os.rename(sFileTemp, sFile)
+    return True
+
+def testVBoxAPI():
+    """
+    Performs various VirtualBox API tests.
+    """
+
+    # Give the user a hint where we gonna install stuff into.
+    if  g_fVerbose \
+    and sys.version_info.major >= 3:
+        import site
+        print("Global site packages directories are:")
+        for sPath in site.getsitepackages():
+            print("\t%s" % (sPath))
+        print("User site packages directories are:")
+        print("\t%s" % (site.getusersitepackages()))
+        print("Module search path is:")
+        for sPath in sys.path:
+            print("\t%s" % (sPath))
+
+    #
+    # Test using the just installed VBox API module by calling some (simpler) APIs
+    # where now kernel drivers are other fancy stuff is needed.
+    #
+    try:
+        from vboxapi import VirtualBoxManager
+        oVBoxMgr = VirtualBoxManager()
+        oVBox    = oVBoxMgr.getVirtualBox()
+        oHost    = oVBox.host
+        if oHost.architecture not in (oVBoxMgr.constants.PlatformArchitecture_x86, \
+                                      oVBoxMgr.constants.PlatformArchitecture_ARM):
+            raise Exception('Host platform invalid!')
+        print("Testing VirtualBox Python bindings successful: Detected VirtualBox %s (%d)" % (oVBox.version, oHost.architecture))
+        _ = oVBox.getMachines()
+        oVBoxMgr.deinit()
+        del oVBoxMgr
+    except ImportError as exc:
+        print("ERROR: Testing VirtualBox Python bindings failed: %s" % (exc))
+        return False
+
+    print("Installation of VirtualBox Python bindings for Python %d.%d successful." \
+          % (sys.version_info.major, sys.version_info.minor))
+    return True
+
+def findModulePathHelper(sModule = 'vboxapi', aDir = sys.path):
+    """
+    Helper function for findModulePath.
+
+    Returns the path found, or None if not found.
+    """
+    for sPath in aDir:
+        if g_fVerbose:
+            print('Searching for "%s" in path "%s" ...' % (sModule, sPath))
+        if os.path.isdir(sPath):
+            aDirEntries = os.listdir(sPath)
+            if g_fVerbose:
+                print(aDirEntries)
+            if sModule in aDirEntries:
+                return os.path.join(sPath, sModule)
+    return None
+
+def findModulePath(sModule = 'vboxapi'):
+    """
+    Finds a module in the system path.
+
+    Returns the path found, or None if not found.
+    """
+    sPath = findModulePathHelper(sModule)
+    if not sPath:
+        try:
+            import site # Might not available everywhere.
+            sPath = findModulePathHelper(sModule, site.getsitepackages())
+        except:
+            pass
+    return sPath
+
+try:
+    from distutils.command.install import install # Only for < Python 3.12.
+except:
+    pass
+
+class setupInstallClass(install):
+    """
+    Class which overrides the "install" command of the setup so that we can
+    run post-install actions.
+    """
+
+    def run(self):
+        def _post_install():
+            if findModulePath():
+                testVBoxAPI()
+        atexit.register(_post_install)
+        install.run(self)
+
+def main():
+    """
+    Main function for the setup script.
+    """
+
+    print("Installing VirtualBox bindings for Python %d.%d ..." % (sys.version_info.major, sys.version_info.minor))
+
+    # Deprecation warning for older Python stuff (< Python 3.x).
+    if sys.version_info.major < 3:
+        print("\nWarning: Running VirtualBox with Python %d.%d is marked as being deprecated.\n" \
+              "Please upgrade your Python installation to avoid breakage.\n" \
+              % (sys.version_info.major, sys.version_info.minor))
+
+    sVBoxInstallPath = os.environ.get("VBOX_MSI_INSTALL_PATH", None)
+    if sVBoxInstallPath is None:
+        sVBoxInstallPath = os.environ.get('VBOX_INSTALL_PATH', None)
+        if sVBoxInstallPath is None:
+            print("No VBOX_INSTALL_PATH defined, exiting")
+            return 1
+
+    sVBoxVersion = os.environ.get("VBOX_VERSION", None)
+    if sVBoxVersion is None:
         # Should we use VBox version for binding module versioning?
-        vboxVersion = "1.0"
-
-    import platform
+        sVBoxVersion = "1.0"
+
+    if g_fVerbose:
+        print("VirtualBox installation directory is: %s" % (sVBoxInstallPath))
 
     if platform.system() == 'Windows':
-        cleanupComCache()
+        cleanupWinComCache()
+
+    # Make sure that we always are in the directory where this script resides.
+    # Otherwise installing packages below won't work.
+    sCurDir = os.path.dirname(os.path.abspath(__file__))
+    if g_fVerbose:
+        print("Current directory is: %s" % (sCurDir))
+    try:
+        os.chdir(sCurDir)
+    except OSError as exc:
+        print("Changing to current directory failed: %s" % (exc))
 
     # Darwin: Patched before installation. Modifying bundle is not allowed, breaks signing and upsets gatekeeper.
     if platform.system() != 'Darwin':
-        vboxSdkDest = os.path.join(vboxDest, "sdk")
-        patchWith(os.path.join(os.path.dirname(sys.argv[0]), 'vboxapi', '__init__.py'), vboxDest, vboxSdkDest)
-
-    setup(name='vboxapi',
-          version=vboxVersion,
-          description='Python interface to VirtualBox',
-          author='Oracle Corp.',
-          author_email='[email protected]',
-          url='https://www.virtualbox.org',
-          packages=['vboxapi']
-          )
+        # @todo r=andy This *will* break the script if VirtualBox installation files will be moved.
+        #              Better would be patching the *installed* module instead of the original module.
+        sVBoxSdkPath = os.path.join(sVBoxInstallPath, "sdk")
+        fRc = patchWith(os.path.join(sCurDir, 'src', 'vboxapi', '__init__.py'), \
+                        sVBoxInstallPath, sVBoxSdkPath)
+        if not fRc:
+            return 1
+
+    try:
+        #
+        # Detect which installation method is being used.
+        #
+        # This is a bit messy due the fact that we want to support a broad range of older and newer
+        # Python versions, along with distributions which maintain their own Python packages (e.g. newer Ubuntus).
+        #
+        fInvokeSetupTools = False
+        if sys.version_info >= (3, 12): # Since Python 3.12 there are no distutils anymore. See PEP632.
+            try:
+                from setuptools import setup
+            except ImportError:
+                print("ERROR: setuptools package not installed, can't continue. Exiting.")
+                return 1
+            setup(cmdclass={"install": setupInstallClass})
+        else:
+            if sys.version_info >= (3, 3): # Starting with Python 3.3 we use pyproject.toml by using setup().
+                try:
+                    from distutils.core import setup # pylint: disable=deprecated-module
+                    setup(cmdclass={"install": setupInstallClass})
+                except ImportError:
+                    print("distutils[.core] package not installed/available, falling back to legacy setuptools ...")
+                    fInvokeSetupTools = True # Invoke setuptools as a last resort.
+            else: # Python 2.7.x + Python < 3.6 legacy cruft.
+                fInvokeSetupTools = True
+
+            if fInvokeSetupTools:
+                if g_fVerbose:
+                    print("Invoking setuptools directly ...")
+                setupTool = setup(name='vboxapi',
+                            version=sVBoxVersion,
+                            description='Python interface to VirtualBox',
+                            author='Oracle Corp.',
+                            author_email='[email protected]',
+                            url='https://www.virtualbox.org',
+                            package_dir={'': 'src'},
+                            packages=['vboxapi'])
+                if setupTool:
+                    sPathInstalled = setupTool.command_obj['install'].install_lib
+                    if sPathInstalled not in sys.path:
+                        print("\nWARNING: Installation path is not in current module search path!")
+                        print("         This might happen on OSes / distributions which only maintain packages by")
+                        print("         a vendor-specific method.")
+                        print("Hints:")
+                        print("- Check how the distribution handles user-installable Python packages.")
+                        print("- Using setuptools directly might be deprecated on the distribution.")
+                        print("- Using \"pip install ./vboxapi\" within a virtual environment (virtualenv)")
+                        print("  might fix this.\n")
+                        sys.path.append(sPathInstalled)
+
+                print("Installed to: %s" % (sPathInstalled))
+
+    except RuntimeError as exc:
+        print("ERROR: Installation of VirtualBox Python bindings failed: %s" % (exc))
+        return 1
+
+    if  fInvokeSetupTools \
+    and not testVBoxAPI():
+        return 1
+    return 0
 
 if __name__ == '__main__':
-    main(sys.argv)
+    sys.exit(main())