Skip to content

Commit

Permalink
daemon, gitpython-developers#525: FIX remote urls in config-files
Browse files Browse the repository at this point in the history
+ Parse most remote & config-urls \-->/.
+ Used relative daemon-paths.
+ Use git-daemon PORT above 10k; on Windows all below need Admin rights.
+FIXED git-daemon  @with_rw_and_rw_remote_repo():
  + test_base.test_with_rw_remote_and_rw_repo() PASS.
  + test_remote.test_base() now freezes! (so still hidden win_err)

+ repo_test: minor finally delete test-repos created inside this repo.
+ util: delete unused `absolute_project_path()`.
  • Loading branch information
ankostis committed Oct 12, 2016
1 parent 8ea7e26 commit 4b586fb
Show file tree
Hide file tree
Showing 9 changed files with 81 additions and 69 deletions.
6 changes: 5 additions & 1 deletion git/cmd.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,14 +186,18 @@ def __setstate__(self, d):
# Override this value using `Git.USE_SHELL = True`
USE_SHELL = False

class AutoInterrupt(object):
@classmethod
def polish_url(cls, url):
return url.replace("\\\\", "\\").replace("\\", "/")

class AutoInterrupt(object):
"""Kill/Interrupt the stored process instance once this instance goes out of scope. It is
used to prevent processes piling up in case iterators stop reading.
Besides all attributes are wired through to the contained process object.
The wait method was overridden to perform automatic status code checking
and possibly raise."""

__slots__ = ("proc", "args")

def __init__(self, proc, args):
Expand Down
4 changes: 2 additions & 2 deletions git/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
join_path,
finalize_process
)
from git.cmd import handle_process_output
from git.cmd import handle_process_output, Git
from gitdb.util import join
from git.compat import (defenc, force_text, is_win)
import logging
Expand Down Expand Up @@ -570,7 +570,7 @@ def create(cls, repo, name, url, **kwargs):
:raise GitCommandError: in case an origin with that name already exists"""
scmd = 'add'
kwargs['insert_kwargs_after'] = scmd
repo.git.remote(scmd, name, url, **kwargs)
repo.git.remote(scmd, name, Git.polish_url(url), **kwargs)
return cls(repo, name)

# add is an alias
Expand Down
2 changes: 1 addition & 1 deletion git/repo/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -897,7 +897,7 @@ def _clone(cls, git, url, path, odb_default_type, progress, **kwargs):
repo = cls(path, odbt=odbt)
if repo.remotes:
with repo.remotes[0].config_writer as writer:
writer.set_value('url', repo.remotes[0].url.replace("\\\\", "\\").replace("\\", "/"))
writer.set_value('url', Git.polish_url(repo.remotes[0].url))
# END handle remote repo
return repo

Expand Down
92 changes: 54 additions & 38 deletions git/test/lib/helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,28 @@
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
from __future__ import print_function

import os
from unittest import TestCase
import time
import tempfile
from functools import wraps
import io
import logging
import os
import tempfile
import textwrap
import time
from unittest import TestCase

from functools import wraps

from git.util import rmtree
from git.compat import string_types, is_win
import textwrap
from git.util import rmtree, HIDE_WINDOWS_KNOWN_ERRORS

import os.path as osp


osp = os.path.dirname
ospd = osp.dirname

GIT_REPO = os.environ.get("GIT_PYTHON_TEST_GIT_REPO_BASE", osp(osp(osp(osp(__file__)))))
GIT_DAEMON_PORT = os.environ.get("GIT_PYTHON_TEST_GIT_DAEMON_PORT", "9418")
GIT_REPO = os.environ.get("GIT_PYTHON_TEST_GIT_REPO_BASE", ospd(ospd(ospd(ospd(__file__)))))
GIT_DAEMON_PORT = os.environ.get("GIT_PYTHON_TEST_GIT_DAEMON_PORT", "19418")

__all__ = (
'fixture_path', 'fixture', 'absolute_project_path', 'StringProcessAdapter',
'fixture_path', 'fixture', 'StringProcessAdapter',
'with_rw_directory', 'with_rw_repo', 'with_rw_and_rw_remote_repo', 'TestBase', 'TestCase',
'GIT_REPO', 'GIT_DAEMON_PORT'
)
Expand All @@ -35,18 +37,13 @@


def fixture_path(name):
test_dir = osp(osp(__file__))
return os.path.join(test_dir, "fixtures", name)
return osp.join(ospd(ospd(__file__)), 'fixtures', name)


def fixture(name):
with open(fixture_path(name), 'rb') as fd:
return fd.read()


def absolute_project_path():
return os.path.abspath(os.path.join(osp(__file__), "..", ".."))

#} END routines

#{ Adapters
Expand Down Expand Up @@ -165,26 +162,31 @@ def repo_creator(self):
return argument_passer


def launch_git_daemon(temp_dir, ip, port):
def launch_git_daemon(base_path, ip, port):
from git import Git
if is_win:
## On MINGW-git, daemon exists in .\Git\mingw64\libexec\git-core\,
# but if invoked as 'git daemon', it detaches from parent `git` cmd,
# and then CANNOT DIE!
# So, invoke it as a single command.
## Cygwin-git has no daemon.
## Cygwin-git has no daemon. But it can use MINGW's.
#
daemon_cmd = ['git-daemon', temp_dir,
daemon_cmd = ['git-daemon',
'--enable=receive-pack',
'--listen=%s' % ip,
'--port=%s' % port]
'--port=%s' % port,
'--base-path=%s' % base_path,
base_path]
gd = Git().execute(daemon_cmd, as_process=True)
else:
gd = Git().daemon(temp_dir,
gd = Git().daemon(base_path,
enable='receive-pack',
listen=ip,
port=port,
base_path=base_path,
as_process=True)
# yes, I know ... fortunately, this is always going to work if sleep time is just large enough
time.sleep(0.5)
return gd


Expand Down Expand Up @@ -212,7 +214,8 @@ def case(self, rw_repo, rw_remote_repo)
See working dir info in with_rw_repo
:note: We attempt to launch our own invocation of git-daemon, which will be shutdown at the end of the test.
"""
from git import Remote, GitCommandError
from git import Git, Remote, GitCommandError

assert isinstance(working_tree_ref, string_types), "Decorator requires ref name for working tree checkout"

def argument_passer(func):
Expand Down Expand Up @@ -240,23 +243,36 @@ def remote_repo_creator(self):
pass
crw.set(section, "receivepack", True)

# initialize the remote - first do it as local remote and pull, then
# we change the url to point to the daemon. The daemon should be started
# by the user, not by us
# Initialize the remote - first do it as local remote and pull, then
# we change the url to point to the daemon.
d_remote = Remote.create(rw_repo, "daemon_origin", remote_repo_dir)
d_remote.fetch()
remote_repo_url = "git://localhost:%s%s" % (GIT_DAEMON_PORT, remote_repo_dir)

base_path, rel_repo_dir = osp.split(remote_repo_dir)

remote_repo_url = "git://localhost:%s/%s" % (GIT_DAEMON_PORT, rel_repo_dir)
with d_remote.config_writer as cw:
cw.set('url', remote_repo_url)

temp_dir = osp(_mktemp())
gd = launch_git_daemon(temp_dir, '127.0.0.1', GIT_DAEMON_PORT)
try:
# yes, I know ... fortunately, this is always going to work if sleep time is just large enough
time.sleep(0.5)
# end

gd = launch_git_daemon(Git.polish_url(base_path), '127.0.0.1', GIT_DAEMON_PORT)
except Exception as ex:
if is_win:
msg = textwrap.dedent("""
The `git-daemon.exe` must be in PATH.
For MINGW, look into .\Git\mingw64\libexec\git-core\), but problems with paths might appear.
CYGWIN has no daemon, but if one exists, it gets along fine (has also paths problems)
Anyhow, alternatively try starting `git-daemon` manually:""")
else:
msg = "Please try starting `git-daemon` manually:"
msg += textwrap.dedent("""
git daemon --enable=receive-pack --base-path=%s %s
You can also run the daemon on a different port by passing --port=<port>"
and setting the environment variable GIT_PYTHON_TEST_GIT_DAEMON_PORT to <port>
""" % (base_path, base_path))
raise AssertionError(ex, msg)
# END make assertion
else:
# try to list remotes to diagnoes whether the server is up
try:
rw_repo.git.ls_remote(d_remote)
Expand All @@ -283,9 +299,9 @@ def remote_repo_creator(self):
git daemon --enable=receive-pack '%s'
You can also run the daemon on a different port by passing --port=<port>"
and setting the environment variable GIT_PYTHON_TEST_GIT_DAEMON_PORT to <port>
""" % temp_dir)
""" % base_path)
from unittest import SkipTest
raise SkipTest(msg) if is_win else AssertionError(msg)
raise SkipTest(msg) if HIDE_WINDOWS_KNOWN_ERRORS else AssertionError(e, msg)
# END make assertion
# END catch ls remote error

Expand Down Expand Up @@ -354,7 +370,7 @@ class TestBase(TestCase):

def _small_repo_url(self):
""":return" a path to a small, clonable repository"""
return os.path.join(self.rorepo.working_tree_dir, 'git/ext/gitdb/gitdb/ext/smmap')
return osp.join(self.rorepo.working_tree_dir, 'git/ext/gitdb/gitdb/ext/smmap')

@classmethod
def setUpClass(cls):
Expand All @@ -378,7 +394,7 @@ def _make_file(self, rela_path, data, repo=None):
with the given data. Returns absolute path to created file.
"""
repo = repo or self.rorepo
abs_path = os.path.join(repo.working_tree_dir, rela_path)
abs_path = osp.join(repo.working_tree_dir, rela_path)
with open(abs_path, "w") as fp:
fp.write(data)
return abs_path
11 changes: 0 additions & 11 deletions git/test/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,17 +110,6 @@ def test_with_rw_repo(self, rw_repo):
assert not rw_repo.config_reader("repository").getboolean("core", "bare")
assert os.path.isdir(os.path.join(rw_repo.working_tree_dir, 'lib'))

# @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, """
# FIXME: helper.wrapper fails with:
# PermissionError: [WinError 5] Access is denied:
# 'C:\\Users\\appveyor\\AppData\\Local\\Temp\\1\\test_work_tree_unsupportedryfa60di\\
# master_repo\\.git\\objects\\pack\\pack-bc9e0787aef9f69e1591ef38ea0a6f566ec66fe3.idx'
# AND
# FIXME: git-daemon failing with:
# git.exc.GitCommandError: Cmd('git') failed due to: exit code(128)
# cmdline: git ls-remote daemon_origin
# stderr: 'fatal: bad config line 15 in file .git/config'
# """)
@with_rw_and_rw_remote_repo('0.1.6')
def test_with_rw_remote_and_rw_repo(self, rw_repo, rw_remote_repo):
assert not rw_repo.config_reader("repository").getboolean("core", "bare")
Expand Down
2 changes: 1 addition & 1 deletion git/test/test_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ def tearDown(self):
import gc
gc.collect()

# @skipIf(HIDE_WINDOWS_KNOWN_ERRORS,
# @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, ## ACTUALLY skipped by `git.submodule.base#L869`.
# "FIXME: helper.wrapper fails with: PermissionError: [WinError 5] Access is denied: "
# "'C:\\Users\\appveyor\\AppData\\Local\\Temp\\1\\test_work_tree_unsupportedryfa60di\\master_repo\\.git\\objects\\pack\\pack-bc9e0787aef9f69e1591ef38ea0a6f566ec66fe3.idx") # noqa E501
@with_rw_directory
Expand Down
19 changes: 9 additions & 10 deletions git/test/test_remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,11 @@
from git.util import IterableList, rmtree
from git.compat import string_types
import tempfile
import os
import os.path as osp
import random
from unittest.case import skipIf
from git.util import HIDE_WINDOWS_KNOWN_ERRORS
from git.cmd import Git

# assure we have repeatable results
random.seed(0)
Expand Down Expand Up @@ -105,7 +108,7 @@ def tearDown(self):
gc.collect()

def _print_fetchhead(self, repo):
with open(os.path.join(repo.git_dir, "FETCH_HEAD")):
with open(osp.join(repo.git_dir, "FETCH_HEAD")):
pass

def _do_test_fetch_result(self, results, remote):
Expand Down Expand Up @@ -156,7 +159,7 @@ def _commit_random_file(self, repo):
# Create a file with a random name and random data and commit it to repo.
# Return the commited absolute file path
index = repo.index
new_file = self._make_file(os.path.basename(tempfile.mktemp()), str(random.random()), repo)
new_file = self._make_file(osp.basename(tempfile.mktemp()), str(random.random()), repo)
index.add([new_file])
index.commit("Committing %s" % new_file)
return new_file
Expand Down Expand Up @@ -263,7 +266,8 @@ def get_info(res, remote, name):
# must clone with a local path for the repo implementation not to freak out
# as it wants local paths only ( which I can understand )
other_repo = remote_repo.clone(other_repo_dir, shared=False)
remote_repo_url = "git://localhost:%s%s" % (GIT_DAEMON_PORT, remote_repo.git_dir)
remote_repo_url = osp.basename(remote_repo.git_dir) # git-daemon runs with appropriate `--base-path`.
remote_repo_url = Git.polish_url("git://localhost:%s/%s" % (GIT_DAEMON_PORT, remote_repo_url))

# put origin to git-url
other_origin = other_repo.remotes.origin
Expand Down Expand Up @@ -384,12 +388,7 @@ def _assert_push_and_pull(self, remote, rw_repo, remote_repo):
TagReference.delete(rw_repo, new_tag, other_tag)
remote.push(":%s" % other_tag.path)

# @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, """
# FIXME: git-daemon failing with:
# git.exc.GitCommandError: Cmd('git') failed due to: exit code(128)
# cmdline: git ls-remote daemon_origin
# stderr: 'fatal: bad config line 15 in file .git/config'
# """)
@skipIf(HIDE_WINDOWS_KNOWN_ERRORS, "FIXME: Freezes!")
@with_rw_and_rw_remote_repo('0.1.6')
def test_base(self, rw_repo, remote_repo):
num_remotes = 0
Expand Down
12 changes: 8 additions & 4 deletions git/test/test_repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -472,12 +472,16 @@ def test_creation_deletion(self):
head = self.rorepo.create_head("new_head", "HEAD~1")
self.rorepo.delete_head(head)

tag = self.rorepo.create_tag("new_tag", "HEAD~2")
self.rorepo.delete_tag(tag)
try:
tag = self.rorepo.create_tag("new_tag", "HEAD~2")
finally:
self.rorepo.delete_tag(tag)
with self.rorepo.config_writer():
pass
remote = self.rorepo.create_remote("new_remote", "git@server:repo.git")
self.rorepo.delete_remote(remote)
try:
remote = self.rorepo.create_remote("new_remote", "git@server:repo.git")
finally:
self.rorepo.delete_remote(remote)

def test_comparison_and_hash(self):
# this is only a preliminary test, more testing done in test_index
Expand Down
2 changes: 1 addition & 1 deletion git/test/test_submodule.py
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ def _do_base_tests(self, rwrepo):
# Error if there is no submodule file here
self.failUnlessRaises(IOError, Submodule._config_parser, rwrepo, rwrepo.commit(self.k_no_subm_tag), True)

# @skipIf(HIDE_WINDOWS_KNOWN_ERRORS,
# @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, ## ACTUALLY skipped by `git.submodule.base#L869`.
# "FIXME: fails with: PermissionError: [WinError 32] The process cannot access the file because"
# "it is being used by another process: "
# "'C:\\Users\\ankostis\\AppData\\Local\\Temp\\tmp95c3z83bnon_bare_test_base_rw\\git\\ext\\gitdb\\gitdb\\ext\\smmap'") # noqa E501
Expand Down

0 comments on commit 4b586fb

Please sign in to comment.