Update manylinux CLI

python_appimage/__main__.py

@@ -59,9 +59,6 @@ def main():
     build_manylinux_parser.add_argument('abi',
         help='python ABI (e.g. cp37-cp37m)')
 
-    build_manylinux_parser.add_argument('--contained', help=argparse.SUPPRESS,
-                                        action='store_true', default=False)
-
     build_app_parser = build_subparsers.add_parser('app',
         description='Build a Python application using a base AppImage')
     build_app_parser.add_argument('appdir',

python_appimage/commands/build/manylinux.py

@@ -1,10 +1,13 @@
 import glob
 import os
+from pathlib import Path
 import platform
 import shutil
 import sys
 
 from ...appimage import build_appimage, relocate_python
+from ...manylinux import Arch, Downloader, ImageExtractor, LinuxTag, \
+                         PythonExtractor
 from ...utils.docker import docker_run
 from ...utils.fs import copy_tree
 from ...utils.manylinux import format_appimage_name, format_tag
@@ -17,7 +20,7 @@ __all__ = ['execute']
 def _unpack_args(args):
     '''Unpack command line arguments
     '''
-    return args.tag, args.abi, args.contained
+    return args.tag, args.abi
 
 
 def _get_appimage_name(abi, tag):
@@ -31,74 +34,35 @@ def _get_appimage_name(abi, tag):
     return format_appimage_name(abi, fullversion, tag)
 
 
-def execute(tag, abi, contained=False):
-    '''Build a Python AppImage using a manylinux docker image
+def execute(tag, abi):
+    '''Build a Python AppImage using a Manylinux image
     '''
 
-    if not contained:
-        # Forward the build to a Docker image
-        image = 'quay.io/pypa/' + format_tag(tag)
-        python = '/opt/python/' + abi + '/bin/python'
-
-        pwd = os.getcwd()
-        dirname = os.path.abspath(os.path.dirname(__file__) + '/../..')
-        with TemporaryDirectory() as tmpdir:
-            copy_tree(dirname, 'python_appimage')
-
-            argv = sys.argv[1:]
-            if argv:
-                argv = ' '.join(argv)
-            else:
-                argv = 'build manylinux {:} {:}'.format(tag, abi)
-            if tag.startswith("1_"):
-                # On manylinux1 tk is not installed
-                script = [
-                    'yum --disablerepo="*" --enablerepo=base install -q -y tk']
-            else:
-                # tk is already installed on other platforms
-                script = []
-            script += [
-                python + ' -m python_appimage ' + argv + ' --contained',
-                ''
-            ]
-            docker_run(image, script)
-
-            appimage_name = _get_appimage_name(abi, tag)
-
-            if tag.startswith('1_') or tag.startswith('2010_'):
-                # appimagetool does not run on manylinux1 (CentOS 5) or
-                # manylinux2010 (CentOS 6). Below is a patch for these specific
-                # cases.
-                arch = tag.split('_', 1)[-1]
-                if arch == platform.machine():
-                    # Pack the image directly from the host
-                    build_appimage(destination=appimage_name)
-                else:
-                    # Use a manylinux2014 Docker image (CentOS 7) in order to
-                    # pack the image.
-                    script = (
-                        'python -m python_appimage ' + argv + ' --contained',
-                        ''
-                    )
-                    docker_run('quay.io/pypa/manylinux2014_' + arch, script)
-
-            shutil.move(appimage_name, os.path.join(pwd, appimage_name))
-
-    else:
-        # We are running within a manylinux Docker image
-        is_manylinux_old = tag.startswith('1_') or tag.startswith('2010_')
-
-        if not os.path.exists('AppDir'):
-            # Relocate the targeted manylinux Python installation
-            relocate_python()
-        else:
-            # This is a second stage build. The Docker image has actually been
-            # overriden (see above).
-            is_manylinux_old = False
-
-        if is_manylinux_old:
-            # Build only the AppDir when running within a manylinux1 Docker
-            # image because appimagetool does not support CentOS 5 or CentOS 6.
-            pass
-        else:
-            build_appimage(destination=_get_appimage_name(abi, tag))
+    tag, arch = tag.split('_', 1)
+    tag = LinuxTag.from_brief(tag)
+    arch = Arch.from_str(arch)
+
+    downloader = Downloader(tag=tag, arch=arch)
+    downloader.download()
+
+    image_extractor = ImageExtractor(downloader.default_destination())
+    image_extractor.extract()
+
+    pwd = os.getcwd()
+    with TemporaryDirectory() as tmpdir:
+        python_extractor = PythonExtractor(
+            arch = arch,
+            prefix = image_extractor.default_destination(),
+            tag = abi
+        )
+        appdir = Path(tmpdir) / 'AppDir'
+        python_extractor.extract(appdir)
+
+        fullname = '-'.join((
+            f'{python_extractor.impl}{python_extractor.version.long()}',
+            abi,
+            f'{tag}_{arch}'
+        ))
+        shutil.move(appdir, os.path.join(pwd, fullname))
+
+        # XXX build_appimage(destination=_get_appimage_name(abi, tag))

python_appimage/manylinux/config.py

@@ -1,6 +1,6 @@
 from enum import auto, Enum
 import platform
-from typing import NamedTuple, Union
+from typing import NamedTuple, Optional, Union
 
 
 __all__ = ['Arch', 'PythonImpl', 'PythonVersion']
@@ -52,11 +52,21 @@ class LinuxTag(Enum):
         else:
             raise NotImplementedError(value)
 
+    @classmethod
+    def from_brief(cls, value) -> 'LinuxTag':
+        if value.startswith('2_'):
+            return cls.from_str('manylinux_' + value)
+        else:
+            return cls.from_str('manylinux' + value)
+
 
 class PythonImpl(Enum):
     '''Supported Python implementations.'''
     CPYTHON = auto()
 
+    def __str__(self):
+        return 'python'
+
 
 class PythonVersion(NamedTuple):
     ''''''
@@ -64,15 +74,33 @@ class PythonVersion(NamedTuple):
     major: int
     minor: int
     patch: Union[int, str]
+    flavour: Optional[str]=None
 
     @classmethod
     def from_str(cls, value: str) -> 'PythonVersion':
         major, minor, patch = value.split('.', 2)
+        try:
+            patch, flavour = patch.split('-', 1)
+        except ValueError:
+            flavour = None
+        else:
+            if flavour == 'nogil':
+                flavour = 't'
+            elif flavour == 'ucs2':
+                flavour = 'm'
+            elif flavour == 'ucs4':
+                flavour = 'mu'
+            else:
+                raise NotImplementedError(value)
         try:
             patch = int(patch)
         except ValueError:
             pass
-        return cls(int(major), int(minor), patch)
+        return cls(int(major), int(minor), patch, flavour)
+
+    def flavoured(self) -> str:
+        flavour = self.flavour if self.flavour == 't' else ''
+        return f'{self.major}.{self.minor}{flavour}'
 
     def long(self) -> str:
         return f'{self.major}.{self.minor}.{self.patch}'

python_appimage/manylinux/extract.py

@@ -12,7 +12,8 @@ import subprocess
 from typing import Dict, List, NamedTuple, Optional, Union
 
 from .config import Arch, PythonImpl, PythonVersion
-from ..utils.deps import ensure_excludelist, EXCLUDELIST
+from ..utils.deps import ensure_excludelist, ensure_patchelf, EXCLUDELIST, \
+                         PATCHELF
 from ..utils.log import debug, log
 
 
@@ -106,17 +107,8 @@ class PythonExtractor:
 
         # Set patchelf, if not provided.
         if self.patchelf is None:
-            paths = (
-                Path(__file__).parent / 'bin',
-                Path.home() / '.local/bin'
-            )
-            for path in paths:
-                patchelf = path / 'patchelf'
-                if patchelf.exists():
-                    break
-            else:
-                raise NotImplementedError()
-            object.__setattr__(self, 'patchelf', patchelf)
+            ensure_patchelf()
+            object.__setattr__(self, 'patchelf', PATCHELF)
         else:
             assert(self.patchelf.exists())
 
@@ -124,6 +116,7 @@ class PythonExtractor:
     def extract(
         self,
         destination: Path,
+        *,
         python_prefix: Optional[str]=None,
         system_prefix: Optional[str]=None
         ):
@@ -131,11 +124,11 @@ class PythonExtractor:
 
         python = f'python{self.version.short()}'
         runtime = f'bin/{python}'
-        packages = f'lib/{python}'
+        packages = f'lib/python{self.version.flavoured()}'
         pip = f'bin/pip{self.version.short()}'
 
         if python_prefix is None:
-            python_prefix = f'opt/{python}'
+            python_prefix = f'opt/python{self.version.flavoured()}'
 
         if system_prefix is None:
             system_prefix = 'usr'
@@ -152,6 +145,8 @@ class PythonExtractor:
             raise NotImplementedError()
 
         # Clone Python runtime.
+        log('CLONE',
+            f'{python} from {self.python_prefix.relative_to(self.prefix)}')
         (python_dest / 'bin').mkdir(exist_ok=True, parents=True)
         shutil.copy(self.python_prefix / runtime, python_dest / runtime)
 
@@ -191,6 +186,7 @@ class PythonExtractor:
                             symlinks=True, dirs_exist_ok=True)
 
         # Remove some clutters.
+        log('PRUNE', '%s packages', python)
         shutil.rmtree(python_dest / packages / 'test', ignore_errors=True)
         for root, dirs, files in os.walk(python_dest / packages):
             root = Path(root)
@@ -226,6 +222,7 @@ class PythonExtractor:
             self.set_rpath(dst, '$ORIGIN')
 
         # Patch RPATHs of binary modules.
+        log('LINK', '%s C-extensions', python)
         path = Path(python_dest / f'{packages}/lib-dynload')
         for module in glob.glob(str(path / "*.so")):
             src = Path(module)
@@ -245,6 +242,7 @@ class PythonExtractor:
             assert(certifi.name == 'certifi')
             site_packages = certifi.parent
             assert(site_packages.name == 'site-packages')
+            log('INSTALL', certifi.name)
 
             for src in glob.glob(str(site_packages / 'certifi*')):
                 src = Path(src)
@@ -264,6 +262,7 @@ class PythonExtractor:
         tx_version.sort()
         tx_version = tx_version[-1]
 
+        log('INSTALL', f'Tcl/Tk{tx_version}')
         tcltk_dir = Path(system_dest / 'share/tcltk')
         tcltk_dir.mkdir(exist_ok=True, parents=True)