Tweak manylinux image extractor

python_appimage/manylinux/download.py

@@ -10,6 +10,7 @@ import tempfile
 from typing import List, Optional
 
 from .config import Arch, LinuxTag
+from ..utils.deps import CACHE_DIR
 from ..utils.log import debug, log
 
 
@@ -52,12 +53,16 @@ class Downloader:
         object.__setattr__(self, 'image', image)
 
 
+    def default_destination(self):
+        return Path(CACHE_DIR) / f'share/images/{self.image}'
+
+
     def download(
         self,
         destination: Optional[Path]=None,
         tag: Optional[str] = 'latest'):
 
-        destination = destination or Path(self.image)
+        destination = destination or self.default_destination()
 
         # Authenticate to quay.io.
         repository = f'pypa/{self.image}'
@@ -88,9 +93,26 @@ class Downloader:
         # Check missing layers to download.
         required = [layer['digest'].split(':', 1)[-1] for layer in
                     manifest['layers']]
-        is_missing = lambda hash_: \
-            not (destination / f'layers/{hash_}.tar.gz').exists()
-        missing = tuple(filter(is_missing, required))
+
+        missing = []
+        for hash_ in required:
+            path = destination / f'layers/{hash_}.tar.gz'
+            if path.exists():
+                hasher = hashlib.sha256()
+                with path.open('rb') as f:
+                    while True:
+                        chunk = f.read(CHUNK_SIZE)
+                        if not chunk:
+                            break
+                        else:
+                            hasher.update(chunk)
+                    h = hasher.hexdigest()
+                    if h != hash_:
+                        missing.append(hash_)
+                    else:
+                        debug('FOUND', f'{hash_}.tar.gz')
+            else:
+                missing.append(hash_)
 
         # Fetch missing layers.
         with tempfile.TemporaryDirectory() as tmpdir:

python_appimage/manylinux/extract.py

@@ -1,3 +1,4 @@
+import atexit
 from dataclasses import dataclass, field
 from distutils.version import LooseVersion
 import glob
@@ -83,6 +84,10 @@ class PythonExtractor:
         if ssl:
             paths.append(Path(ssl[0]) / 'lib')
 
+        mpdecimal = glob.glob(str(self.prefix / 'opt/_internal/mpdecimal-*'))
+        if mpdecimal:
+            paths.append(Path(mpdecimal[0]) / 'lib')
+
         object.__setattr__(self, 'library_path', paths)
 
         # Set excluded libraries.
@@ -307,21 +312,56 @@ class ImageExtractor:
     '''Manylinux image tag.'''
 
 
-    def extract(self, destination: Path):
+    def default_destination(self):
+        return self.prefix / f'extracted/{self.tag}'
+
+
+    def extract(self, destination: Optional[Path]=None, *, cleanup=False):
         '''Extract Manylinux image.'''
 
+        if destination is None:
+            destination = self.default_destination()
+
+        if cleanup:
+            def cleanup(destination):
+                shutil.rmtree(destination, ignore_errors=True)
+            atexit.register(cleanup, destination)
+
         with open(self.prefix / f'tags/{self.tag}.json') as f:
             meta = json.load(f)
         layers = meta['layers']
 
-        for layer in layers:
-            debug('EXTRACT', f'{layer}.tar.gz')
+        extracted = []
+        extracted_file = destination / '.extracted'
+        if destination.exists():
+            clean_destination = True
+            if extracted_file.exists():
+                with extracted_file.open() as f:
+                    extracted = f.read().split(os.linesep)[:-1]
+
+                for a, b in zip(layers, extracted):
+                    if a != b:
+                        break
+                else:
+                    clean_destination = False
+
+            if clean_destination:
+                shutil.rmtree(destination, ignore_errors=True)
+
+        for i, layer in enumerate(layers):
+            try:
+                if layer == extracted[i]:
+                    continue
+            except IndexError:
+                pass
 
+            debug('EXTRACT', f'{layer}.tar.gz')
             filename = self.prefix / f'layers/{layer}.tar.gz'
-            cmd = ' && '.join((
-                 f'mkdir -p {destination}',
-                 f'tar -xzf {filename} -C {destination}',
-                 f'chmod u+rw -R {destination}'
+            cmd = ''.join((
+                 f'trap \'chmod u+rw -R {destination}\' EXIT ; ',
+                 f'mkdir -p {destination} && ',
+                 f'tar -xzf {filename} -C {destination} && ',
+                 f'echo \'{layer}\' >> {extracted_file}'
             ))
-            process = subprocess.run(cmd, shell=True, check=True,
-                                     capture_output=True)
+            process = subprocess.run(f'/bin/bash -c "{cmd}"', shell=True,
+                                     check=True, capture_output=True)

python_appimage/utils/deps.py

@@ -11,22 +11,22 @@ from .url import urlretrieve
 
 _ARCH = platform.machine()
 
-_CACHE_DIR = os.path.expanduser('~/.cache/python-appimage')
-
+CACHE_DIR = os.path.expanduser('~/.cache/python-appimage')
+'''Package cache location'''
 
 PREFIX = os.path.abspath(os.path.dirname(__file__) + '/..')
 '''Package installation prefix'''
 
-APPIMAGETOOL_DIR = os.path.join(_CACHE_DIR, 'bin')
+APPIMAGETOOL_DIR = os.path.join(CACHE_DIR, 'bin')
 '''Location of the appimagetool binary'''
 
 APPIMAGETOOL_VERSION = '12'
 '''Version of the appimagetool binary'''
 
-EXCLUDELIST = os.path.join(_CACHE_DIR, 'share/excludelist')
+EXCLUDELIST = os.path.join(CACHE_DIR, 'share/excludelist')
 '''AppImage exclusion list'''
 
-PATCHELF = os.path.join(_CACHE_DIR, 'bin/patchelf')
+PATCHELF = os.path.join(CACHE_DIR, 'bin/patchelf')
 '''Location of the PatchELF binary'''
 
 PATCHELF_VERSION = '0.14.3'
@@ -93,7 +93,7 @@ def ensure_patchelf():
     if os.path.exists(PATCHELF):
         return False
 
-    tgz = '-'.join(('patchelf', _PATCHELF_VERSION, _ARCH)) + '.tar.gz'
+    tgz = '-'.join(('patchelf', PATCHELF_VERSION, _ARCH)) + '.tar.gz'
     baseurl = 'https://github.com/NixOS/patchelf'
     log('INSTALL', 'patchelf from %s', baseurl)
 
@@ -102,7 +102,7 @@ def ensure_patchelf():
     make_tree(dirname)
     with TemporaryDirectory() as tmpdir:
         urlretrieve(os.path.join(baseurl, 'releases', 'download',
-                                 _PATCHELF_VERSION, tgz), tgz)
+                                 PATCHELF_VERSION, tgz), tgz)
         system(('tar', 'xzf', tgz))
         copy_file('bin/patchelf', patchelf)
     os.chmod(patchelf, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)

python_appimage/utils/url.py

@@ -32,9 +32,9 @@ def urlretrieve(url, filename=None):
     else:
         debug('DOWNLOAD', '%s as %s', url, filename)
 
-    parent_directory = os.path.dirname(filename)
-    if not os.path.exists(parent_directory):
-        os.makedirs(parent_directory)
+        parent_directory = os.path.dirname(filename)
+        if parent_directory and not os.path.exists(parent_directory):
+            os.makedirs(parent_directory)
 
     if _urlretrieve is None:
         data = urllib2.urlopen(url).read()