Skip to content

Commit 99b35a1

Browse files
zraxzrax
authored
Merge pull request #209 from dotjrich/pymultic-container-build
Adds support for building our own Python containers
2 parents 7869a08 + 9031029 commit 99b35a1

File tree

3 files changed

+139
-25
lines changed

3 files changed

+139
-25
lines changed

‎scripts/Dockerfile.pybuild

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#
2+
# This will generate a local container with the specified version of Python.
3+
#
4+
# Note: This assumes the the tarball for the particular version you'd like to build has already been downloaded,
5+
# unpacked, and patched appropriately.
6+
#
7+
# To build:
8+
# docker build --build-arg python_version=PYTHON-VERSION \
9+
# --build-arg install_target=INSTALL-TARGET \
10+
# -f Dockerfile.pybuild \
11+
# -t python:PYTHON-VERSION
12+
#
13+
# PYTHON-VERSION full version of Python to build (ex. 3.0.1)
14+
# INSTALL-TARGET the target to make for installing (fullinstall for 3.0.1, install otherwise)
15+
#
16+
# Example for 3.0.1:
17+
# docker build --build-arg python_version=3.0.1 \
18+
# --build-arg install_target=fullinstall \
19+
# -f Dockerfile.pybuild \
20+
# -t python:3.0.1
21+
#
22+
23+
FROM debian:buster
24+
25+
ARG python_version
26+
ARG install_target
27+
28+
RUN mkdir -p /pybuild/ &&\
29+
apt-get update && \
30+
apt-get upgrade -y && \
31+
apt-get install -y build-essential libncurses-dev libz-dev libbz2-dev libreadline-dev
32+
33+
COPY Python-$python_version /pybuild/
34+
35+
RUN cd pybuild &&\
36+
./configure && \
37+
make && \
38+
make $install_target && \
39+
cd / && \
40+
apt-get remove -y build-essential libncurses-dev libz-dev libbz2-dev libreadline-dev && \
41+
apt autoremove -y && \
42+
rm -rf /pybuild

‎scripts/pymultic

Lines changed: 81 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,20 @@ CONTAINER_EXE_EXTRA_ARGS = {
6060
}
6161

6262

63+
def get_container_exe():
64+
container_exe = None
65+
for ce in CONTAINER_EXES:
66+
if shutil.which(ce) is not None:
67+
container_exe = ce
68+
break
69+
70+
if container_exe is None:
71+
print('Cannot find {} in $PATH'.format(' or '.join(CONTAINER_EXES)))
72+
return sys.exit(1)
73+
74+
return container_exe
75+
76+
6377
def fetch_python(snekdir, version):
6478
realver = PYVERS[version]
6579
if version in ('1.0', '1.1'):
@@ -142,6 +156,35 @@ def acquire_python(snekdir, version):
142156
return pyexe
143157

144158

159+
def build_python_container(snekdir, version):
160+
realver = PYVERS[version]
161+
snek = 'Python-{}'.format(realver)
162+
builddir = os.path.join(snekdir, snek)
163+
container_exe = get_container_exe()
164+
py_container_tag = 'python:{}'.format(realver)
165+
166+
if subprocess.call([container_exe, 'image', 'inspect', py_container_tag],
167+
stdout=subprocess.DEVNULL,
168+
stderr=subprocess.DEVNULL) == 0:
169+
return
170+
171+
fetch_python(snekdir, version)
172+
173+
print('Building Python {} container...'.format(realver))
174+
logfile = os.path.join(snekdir, '{}.build.log'.format(snek))
175+
install_target = 'fullinstall' if version == '3.0' else 'install'
176+
with open(logfile, 'wb') as log:
177+
if subprocess.call([container_exe, 'build',
178+
'--build-arg', 'python_version={}'.format(realver),
179+
'--build-arg', 'install_target={}'.format(install_target),
180+
'-f', os.path.join(snekdir, 'Dockerfile.pybuild'),
181+
'-t', py_container_tag, snekdir], stdout=log, stderr=log) != 0:
182+
print('...Container build failed. See {} for details'.format(logfile))
183+
sys.exit(1)
184+
185+
shutil.rmtree(builddir)
186+
187+
145188
def local_compile(snekdir, ver, infile):
146189
pyexe = acquire_python(snekdir, ver)
147190
proc = subprocess.Popen([pyexe, '-c', 'import sys; print(sys.version)'],
@@ -194,24 +237,14 @@ def local_compile(snekdir, ver, infile):
194237
return outfile
195238

196239

197-
def container_compile(ver, infile):
240+
def container_compile(snekdir, ver, infile):
198241
if ver not in PYVER_CONTAINERS:
199-
print('Container compilation not supported for version {}'.format(ver))
200-
return None
201-
202-
container_exe = None
203-
for ce in CONTAINER_EXES:
204-
if shutil.which(ce) is not None:
205-
container_exe = ce
206-
break
207-
208-
if container_exe is None:
209-
print('Cannot find {} in $PATH'.format(' or '.join(CONTAINER_EXES)))
210-
return None
242+
build_python_container(snekdir, ver)
211243

244+
container_exe = get_container_exe()
212245
fullver = PYVERS[ver]
213-
214246
indir = os.path.dirname(os.path.abspath(infile))
247+
infile_full_path = infile
215248
infile = os.path.basename(infile)
216249

217250
if infile.endswith('.py'):
@@ -223,14 +256,38 @@ def container_compile(ver, infile):
223256
os.unlink(outfile)
224257

225258
print('*** Compiling for Python {}'.format(fullver))
226-
proc = subprocess.Popen([container_exe, 'run'] + CONTAINER_EXE_EXTRA_ARGS[container_exe] +
227-
['--rm', '--name', '{}'.format(outfile),
228-
'-v', '{}:/indir:Z'.format(indir),
229-
'-v', '{}:/outdir:Z'.format(os.getcwd()), '-w', '/outdir',
230-
'python:{}'.format(fullver),
231-
'python', '-c',
232-
"import py_compile; py_compile.compile('/indir/{}', '{}')".format(infile, outfile)])
233-
proc.communicate()
259+
if ver in {'1.0', '1.1', '1.2', '1.3', '1.4'}:
260+
# The hard way -- hope your code is safe...
261+
comptmp = os.path.join(indir, 'pymc_temp.py')
262+
if os.path.exists(comptmp):
263+
os.unlink(comptmp)
264+
shutil.copyfile(infile_full_path, comptmp)
265+
proc = subprocess.Popen([container_exe, 'run'] + CONTAINER_EXE_EXTRA_ARGS[container_exe] +
266+
['--rm', '--name', outfile,
267+
'-v', '{}:/indir:Z'.format(indir),
268+
'-v', '{}:/outdir:Z'.format(os.getcwd()), '-w', '/outdir',
269+
'-w', '/indir',
270+
'python:{}'.format(fullver),
271+
'python', '-c',
272+
"import pymc_temp"])
273+
proc.communicate()
274+
if os.path.exists(comptmp + 'o'):
275+
shutil.copyfile(comptmp + 'o', outfile)
276+
os.unlink(comptmp + 'o')
277+
elif os.path.exists(comptmp + 'c'):
278+
shutil.copyfile(comptmp + 'c', outfile)
279+
os.unlink(comptmp + 'c')
280+
os.unlink(comptmp)
281+
else:
282+
# The easy way
283+
proc = subprocess.Popen([container_exe, 'run'] + CONTAINER_EXE_EXTRA_ARGS[container_exe] +
284+
['--rm', '--name', outfile,
285+
'-v', '{}:/indir:Z'.format(indir),
286+
'-v', '{}:/outdir:Z'.format(os.getcwd()), '-w', '/outdir',
287+
'python:{}'.format(fullver),
288+
'python', '-c',
289+
"import py_compile; py_compile.compile('/indir/{}', '{}')".format(infile, outfile)])
290+
proc.communicate()
234291

235292
return outfile
236293

@@ -282,12 +339,11 @@ result = 0
282339
for ver in pythons:
283340
compile_with_container = use_containers
284341
if use_containers and ver not in PYVER_CONTAINERS:
285-
print('Warning: No supported container for {} - using local build'.format(ver))
286-
compile_with_container = False
342+
print('Warning: No officially supported container for {} - using locally built one'.format(ver))
287343

288344
outfile = None
289345
if compile_with_container:
290-
outfile = container_compile(ver, infile)
346+
outfile = container_compile(snekdir, ver, infile)
291347
else:
292348
outfile = local_compile(snekdir, ver, infile)
293349

‎scripts/python-builds/Python-3.1.5.patch

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
diff --git a/Makefile.pre.in b/Makefile.pre.in
2+
index e01cd2745a..dcf465e30e 100644
3+
--- a/Makefile.pre.in
4+
+++ b/Makefile.pre.in
5+
@@ -817,6 +817,11 @@ bininstall: altbininstall
6+
else true; \
7+
fi
8+
(cd $(DESTDIR)$(BINDIR); $(LN) python$(VERSION)$(EXE) $(PYTHON)3$(EXE))
9+
+ -if test -f $(DESTDIR)$(BINDIR)/$(PYTHON)$(EXE) -o -h $(DESTDIR)$(BINDIR)/$(PYTHON)$(EXE); \
10+
+ then rm -f $(DESTDIR)$(BINDIR)/$(PYTHON)$(EXE); \
11+
+ else true; \
12+
+ fi
13+
+ (cd $(DESTDIR)$(BINDIR); $(LN) python$(VERSION)$(EXE) $(PYTHON)$(EXE))
14+
-rm -f $(DESTDIR)$(BINDIR)/python3-config
15+
(cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-config python3-config)
16+
-rm -f $(DESTDIR)$(LIBPC)/python3.pc

0 commit comments

Comments
 (0)