--- a/src/modules/misc.py Tue Aug 18 17:39:20 2015 -0700
+++ b/src/modules/misc.py Wed Aug 19 14:07:41 2015 -0700
@@ -172,12 +172,14 @@
os.utime(d_path, (s.st_atime, s.st_mtime))
elif S_ISSOCK(s.st_mode):
sock = socket.socket(socket.AF_UNIX)
+ # os.mknod doesn't work correctly in 64 bit.
+ run_bit = struct.calcsize("P") * 8
# The s11 fcs version of python doesn't have os.mknod()
# but sock.bind has a path length limitation that we can
# hit when archiving the test suite.
# E1101 Module '{0}' has no '{1}' member
# pylint: disable=E1101
- if hasattr(os, "mknod"):
+ if hasattr(os, "mknod") and run_bit == 32:
os.mknod(d_path, s.st_mode, s.st_dev)
else:
try:
@@ -2888,3 +2890,21 @@
return 1
return NotImplemented
+def set_memory_limit(bytes, allow_override=True):
+ """Limit memory consumption of current process to 'bytes'."""
+
+ if allow_override:
+ try:
+ bytes = int(os.environ["PKG_CLIENT_MAX_PROCESS_SIZE"])
+ except (KeyError, ValueError):
+ pass
+
+ try:
+ resource.setrlimit(resource.RLIMIT_DATA, (bytes, bytes))
+ except AttributeError:
+ # If platform doesn't support RLIMIT_DATA, just ignore it.
+ pass
+ except ValueError:
+ # An unprivileged user can not raise a previously set limit,
+ # if that ever happens, just ignore it.
+ pass
--- a/src/pull.py Tue Aug 18 17:39:20 2015 -0700
+++ b/src/pull.py Wed Aug 19 14:07:41 2015 -0700
@@ -401,6 +401,9 @@
gettext.install("pkg", "/usr/share/locale",
codeset=locale.getpreferredencoding())
+ # set process limits for memory consumption to 8GB
+ misc.set_memory_limit(8 * 1024 * 1024 * 1024)
+
global_settings.client_name = "pkgrecv"
target = os.environ.get("PKG_DEST", None)
src_uri = os.environ.get("PKG_SRC", None)
--- a/src/setup.py Tue Aug 18 17:39:20 2015 -0700
+++ b/src/setup.py Wed Aug 19 14:07:41 2015 -0700
@@ -105,6 +105,14 @@
assert py_version in ('2.7', '3.4')
py_install_dir = 'usr/lib/python' + py_version + '/vendor-packages'
+py64_executable = None
+#Python 3 is always 64 bit and located in /usr/bin.
+if float(py_version) < 3 and osname == 'sunos':
+ if arch == 'sparc':
+ py64_executable = '/usr/bin/sparcv9/python' + py_version
+ elif arch == 'i386':
+ py64_executable = '/usr/bin/amd64/python' + py_version
+
scripts_dir = 'usr/bin'
lib_dir = 'usr/lib'
svc_method_dir = 'lib/svc/method'
@@ -736,7 +744,8 @@
dst_path = util.change_root(self.root_dir,
os.path.join(d, dstname))
dir_util.mkpath(dst_dir, verbose=True)
- file_util.copy_file(srcname, dst_path, update=True)
+ file_util.copy_file(srcname, dst_path,
+ update=True)
# make scripts executable
os.chmod(dst_path,
os.stat(dst_path).st_mode
@@ -838,6 +847,10 @@
cddl_re = re.compile("\n(#\s*\n)?^[^\n]*CDDL HEADER START.+"
"CDDL HEADER END[^\n]*$(\n#\s*$)?", re.MULTILINE|re.DOTALL)
+ # Look for shebang line to replace with arch-specific Python executable.
+ shebang_re = re.compile('^#!.*python[0-9]\.[0-9]')
+ first_buf = True
+
with open(src, "r") as sfp:
try:
os.unlink(dst)
@@ -865,9 +878,21 @@
blanks = "#\n" * count
buf = cddl_re.sub("\n" + blanks,
buf)
+
+ if not first_buf or not py64_executable:
+ dfp.write(buf)
+ continue
+
+ fl = buf[:buf.find(os.linesep) + 1]
+ sb_match = shebang_re.search(fl)
+ if sb_match:
+ buf = shebang_re.sub(
+ "#!" + py64_executable,
+ buf)
else:
buf = cddl_re.sub("", buf)
dfp.write(buf)
+ first_buf = False
# Make file_util use our version of _copy_file_contents
file_util._copy_file_contents = _copy_file_contents
--- a/src/tests/api/t_misc.py Tue Aug 18 17:39:20 2015 -0700
+++ b/src/tests/api/t_misc.py Wed Aug 19 14:07:41 2015 -0700
@@ -33,6 +33,7 @@
import os
import shutil
import stat
+import subprocess
import sys
import tempfile
import unittest
@@ -115,6 +116,86 @@
libc = ctypes.CDLL('libc.so')
self.assertEqual(psinfo.pr_zoneid, libc.getzoneid())
+ def test_memory_limit(self):
+ """Verify that set_memory_limit works."""
+
+ # memory limit to test, keep small to avoid test slowdown
+ mem_cap = 100 * 1024 * 1024
+ # memory tolerance: allowed discrepancy between set limit and
+ # measured process resources. Note, that we have a static
+ # overhead in waste.py for the forking of ps, so while 20M seems
+ # large compared to a 100M limit, in a real world example with
+ # 8G limit it's fairly small.
+ mem_tol = 20 * 1024 * 1024
+
+ waste_mem_py = """
+import os
+import resource
+import subprocess
+
+import pkg.misc as misc
+
+misc.set_memory_limit({0})
+i = 0
+x = {{}}
+try:
+ while True:
+ i += 1
+ x[i] = range(i)
+except MemoryError:
+ # give us some breathing room (enough so the test with env var works)
+ misc.set_memory_limit({0} * 3, allow_override=False)
+ print subprocess.check_output(['ps', '-o', 'rss=', '-p',
+ str(os.getpid())]).strip()
+""".format(str(mem_cap))
+
+ # Re-setting limits which are higher than original limit can
+ # only be done by root.
+ self.assertTrue(os.geteuid() == 0,
+ "must be root to run this test")
+
+ tmpdir = tempfile.mkdtemp(dir=self.test_root)
+ tmpfile = os.path.join(tmpdir, 'waste.py')
+ with open(tmpfile, 'w') as f:
+ f.write(waste_mem_py)
+
+ res = int(subprocess.check_output(['python2.7', tmpfile]))
+ # convert from kB to bytes
+ res *= 1024
+
+ self.debug("mem_cap: " + str(mem_cap))
+ self.debug("proc size: " + str(res))
+
+ self.assertTrue(res < mem_cap + mem_tol,
+ "process mem consumption too high")
+ self.assertTrue(res > mem_cap - mem_tol,
+ "process mem consumption too low")
+
+ # test if env var works
+ os.environ["PKG_CLIENT_MAX_PROCESS_SIZE"] = str(mem_cap * 2)
+ res = int(subprocess.check_output(['python2.7', tmpfile]))
+ res *= 1024
+
+ self.debug("mem_cap: " + str(mem_cap))
+ self.debug("proc size: " + str(res))
+
+ self.assertTrue(res < mem_cap * 2 + mem_tol,
+ "process mem consumption too high")
+ self.assertTrue(res > mem_cap * 2 - mem_tol,
+ "process mem consumption too low")
+
+ # test if invalid env var is handled correctly
+ os.environ["PKG_CLIENT_MAX_PROCESS_SIZE"] = "octopus"
+ res = int(subprocess.check_output(['python2.7', tmpfile]))
+ res *= 1024
+
+ self.debug("mem_cap: " + str(mem_cap))
+ self.debug("proc size: " + str(res))
+
+ self.assertTrue(res < mem_cap + mem_tol,
+ "process mem consumption too high")
+ self.assertTrue(res > mem_cap - mem_tol,
+ "process mem consumption too low")
if __name__ == "__main__":
unittest.main()
--- a/src/tests/cli/t_pkg_install.py Tue Aug 18 17:39:20 2015 -0700
+++ b/src/tests/cli/t_pkg_install.py Wed Aug 19 14:07:41 2015 -0700
@@ -4011,15 +4011,6 @@
sock = socket.socket(socket.AF_UNIX)
sock.bind(os.path.join(self.img_path(), "salvage", "socket"))
sock.close()
- # We also test block and character special files, but only if
- # os.mknod() is available, which it isn't always.
- # Since mknod only supports 32-bit integer currently, we have
- # to check if we are running in 32-bit.
- run_bit = struct.calcsize("P") * 8
- if hasattr(os, "mknod") and run_bit == 32:
- st = os.stat("/dev/null")
- os.mknod(os.path.join(self.img_path(), "salvage",
- "node"), st.st_mode, st.st_dev)
# This could hang reading fifo, or keel over reading socket.
self.pkg("uninstall salvage-special")