Merge pull request #381 from vbatts/vbatts-xattr_support

tarfile: pax header and xattr support

docker_registry/images.py

@@ -3,7 +3,6 @@
 import datetime
 import functools
 import logging
-import tarfile
 import time
 
 import flask
@@ -20,6 +19,9 @@
 from .lib import checksums
 from .lib import layers
 from .lib import mirroring
+# this is our monkey patched snippet from python v2.7.6 'tarfile'
+# with xattr support
+from .lib.xtarfile import tarfile
 
 
 store = storage.load()

docker_registry/lib/layers.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 import logging
-import tarfile
 import tempfile
 
 import backports.lzma as lzma
@@ -11,6 +10,10 @@
 from .. import storage
 from . import cache
 from . import rqueue
+# this is our monkey patched snippet from python v2.7.6 'tarfile'
+# with xattr support
+from .xtarfile import tarfile
+
 
 store = storage.load()
 

docker_registry/lib/xtarfile.py

@@ -0,0 +1,81 @@
+'''
+This is __proc_pax from ./Lib/tarfile.py from v2.7.6
+catching raw (non-utf8) bytes to support some xattr headers in tar archives
+
+This is for the use-case of reading the tar archive, not for the use case of
+interacting with inodes on the filesystem that have xattr's.
+ -- vbatts
+'''
+
+import re
+import tarfile
+
+
+def _proc_pax(self, filetar):
+    """Process an extended or global header as described in
+      POSIX.1-2001.
+   """
+    # Read the header information.
+    buf = filetar.fileobj.read(self._block(self.size))
+
+    # A pax header stores supplemental information for either
+    # the following file (extended) or all following files
+    # (global).
+    if self.type == tarfile.XGLTYPE:
+        pax_headers = filetar.pax_headers
+    else:
+        pax_headers = filetar.pax_headers.copy()
+
+    # Parse pax header information. A record looks like that:
+    # "%d %s=%s\n" % (length, keyword, value). length is the size
+    # of the complete record including the length field itself and
+    # the newline. keyword and value are both UTF-8 encoded strings.
+    regex = re.compile(r"(\d+) ([^=]+)=", re.U)
+    pos = 0
+    while True:
+        match = regex.match(buf, pos)
+        if not match:
+            break
+
+        length, keyword = match.groups()
+        length = int(length)
+        value = buf[match.end(2) + 1:match.start(1) + length - 1]
+
+        try:
+            keyword = keyword.decode("utf8")
+        except Exception:
+            # just leave the raw bytes
+            pass
+
+        try:
+            value = value.decode("utf8")
+        except Exception:
+            # just leave the raw bytes
+            pass
+
+        pax_headers[keyword] = value
+        pos += length
+
+    # Fetch the next header.
+    try:
+        next = self.fromtarfile(filetar)
+    except tarfile.HeaderError:
+        raise tarfile.SubsequentHeaderError("missing or bad subsequent header")
+
+    if self.type in (tarfile.XHDTYPE, tarfile.SOLARIS_XHDTYPE):
+        # Patch the TarInfo object with the extended header info.
+        next._apply_pax_info(pax_headers, filetar.encoding, filetar.errors)
+        next.offset = self.offset
+
+        if "size" in pax_headers:
+            # If the extended header replaces the size field,
+            # we need to recalculate the offset where the next
+            # header starts.
+            offset = next.offset_data
+            if next.isreg() or next.type not in tarfile.SUPPORTED_TYPES:
+                offset += next._block(next.size)
+            filetar.offset = offset
+
+    return next
+
+tarfile.TarInfo._proc_pax = _proc_pax

tests/base.py

@@ -3,12 +3,15 @@
 import docker_registry.run as run
 
 import hashlib
+import os
 import random
 import string
 import unittest
 
 from docker_registry.core import compat
 
+data_dir = os.path.join(os.path.dirname(__file__), "data")
+
 
 class TestCase(unittest.TestCase):
 

tests/data/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json

@@ -0,0 +1 @@
+{"id":"46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457","parent":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","created":"2014-04-07T02:45:52.610504484Z","container":"e0f07f8d72cae171a3dcc35859960e7e956e0628bce6fedc4122bf55b2c287c7","container_config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","sed -ri 's/^(%wheel.*)(ALL)$/\\1NOPASSWD: \\2/' /etc/sudoers"],"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.9.1-dev","config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":3425}

tests/data/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json

@@ -0,0 +1 @@
+{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}

tests/data/xattr/json

@@ -0,0 +1 @@
+{"id":"4439c3c7f847954100b42b267e7e5529cac1d6934db082f65795c5ca2e594d93","parent":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","created":"2014-05-16T17:19:44.091534414Z","container":"5f92fb06cc58f357f0cde41394e2bbbb664e663974b2ac1693ab07b7a306749b","container_config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","setcap 'cap_setgid,cap_setuid+ep' ./file \u0026\u0026 getcap ./file"],"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.11.1-dev","config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":0}

tests/test_tarfile.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+
+import imp
+import os
+
+from nose import tools
+
+import base
+from docker_registry.lib import checksums
+from docker_registry.lib import xtarfile
+
+
+# setting like this in test, due to flake8 H302
+tarfile = xtarfile.tarfile
+
+# To test whether the UnicodeDecodeError still exists
+# (it's still present in python 3.4.0)
+# ((loading this way, since we've monkey patched currently loaded tarfile))
+tarfile_vanilla = imp.load_module('test_failing', *imp.find_module('tarfile'))
+
+
+class TestTarfile(base.TestCase):
+    @tools.raises(UnicodeDecodeError)
+    def test_vanilla_tarfile(self):
+        layer_fh = open(os.path.join(base.data_dir, "xattr/layer.tar"))
+        tar = tarfile_vanilla.open(mode='r|*', fileobj=layer_fh)
+        assert tar
+
+    def test_headers(self):
+        expected = {
+            "46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar": {  # noqa
+                "dev": {
+                    "headers": {"size": 0, "mode": 0o40755, "type": "5"},
+                    "pax": {},
+                },
+                "dev/core": {
+                    "headers": {"size": 0, "mode": 0o120777, "type": "2"},
+                    "pax": {},
+                },
+                "dev/stderr": {
+                    "headers": {"size": 0, "mode": 0o120777, "type": "2"},
+                    "pax": {},
+                },
+                "dev/stdout": {
+                    "headers": {"size": 0, "mode": 0o120777, "type": "2"},
+                    "pax": {},
+                },
+                "dev/fd": {
+                    "headers": {"size": 0, "mode": 0o120777, "type": "2"},
+                    "pax": {},
+                },
+                "dev/ptmx": {
+                    "headers": {"size": 0, "mode": 0o120777, "type": "2"},
+                    "pax": {},
+                },
+                "dev/stdin": {
+                    "headers": {"size": 0, "mode": 0o120777, "type": "2"},
+                    "pax": {},
+                },
+                "etc": {
+                    "headers": {"size": 0, "mode": 0o40755, "type": "5"},
+                    "pax": {},
+                },
+                "etc/sudoers": {
+                    "headers": {"size": 3348, "mode": 0o100440, "type": "0"},
+                    "pax": {},
+                },
+            },
+            "511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar": {  # noqa
+                ".": {
+                    "headers": {"size": 0, "mode": 0o40755, "type": "5"},
+                    "pax": {},
+                },
+            },
+            "xattr/layer.tar": {
+                "file": {
+                    "headers": {"size": 0, "mode": 0o100644, "type": "0"},
+                    "pax": {u"SCHILY.xattr.security.capability": "\x01\x00\x00\x02\xc0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"},  # noqa
+                },
+            },
+        }
+        for file in expected.keys():
+            layer_fh = open(os.path.join(base.data_dir, file))
+            tar = tarfile.open(mode='r|*', fileobj=layer_fh)
+            member_count = 0
+            for member in tar:
+                member_count += 1
+                # check that we know the file names
+                msg = "in %s, did not find file %s" % (file, member.path)
+                l = len(filter(lambda x: member.path in x,
+                        expected[file].keys()))
+                assert (l > 0), msg
+                e = expected[file][member.path]
+                for attr in e["headers"].keys():
+                    msg = "in %s:%s, expected %s of %s, but got %s" % (
+                        file, member.path, attr, e["headers"][attr],
+                        getattr(member, attr))
+                    assert e["headers"][attr] == getattr(member, attr), msg
+                for attr in e["pax"].keys():
+                    msg = b"in %s:%s, expected %s of %s, but got %s".format(
+                        file, member.path, attr, e["pax"][attr],
+                        member.pax_headers[attr])
+                    assert e["pax"][attr] == member.pax_headers[attr], msg
+
+            assert member_count == len(expected[file])
+            layer_fh.close()
+
+    def test_tarsum(self):
+        expected = {
+            "46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457": "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",  # noqa
+            "511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158": "tarsum+sha256:ac672ee85da9ab7f9667ae3c32841d3e42f33cc52c273c23341dabba1c8b0c8b",  # noqa
+            "xattr": "tarsum+sha256:e86f81a4d552f13039b1396ed03ca968ea9717581f9577ef1876ea6ff9b38c98",  # noqa
+        }
+        for layer in expected.keys():
+            layer_fh = open(os.path.join(base.data_dir, layer, "layer.tar"))
+            json_fh = open(os.path.join(base.data_dir, layer, "json"))
+
+            tarsum = checksums.TarSum(json_fh.read())
+            tar = tarfile.open(mode='r|*', fileobj=layer_fh)
+            for member in tar:
+                tarsum.append(member, tar)
+            sum = tarsum.compute()
+            msg = "layer %s, expected [%s] but got [%s]" % (
+                layer, expected[layer], sum)
+            assert expected[layer] == sum, msg
+
+            layer_fh.close()
+            json_fh.close()