16428119 pkg/depot should use existing indexes and respect writable_root
17343539 pkg/depot doesn't support repository configuration retrieval
17564887 Apache depot leaking empty directories in /tmp
--- a/src/depot-config.py Tue Nov 26 11:03:16 2013 +1300
+++ b/src/depot-config.py Wed Dec 04 15:55:52 2013 +1300
@@ -32,6 +32,7 @@
import os
import re
import shutil
+import simplejson as json
import socket
import sys
import traceback
@@ -71,6 +72,7 @@
DEPOT_HTDOCS_DIRNAME = "htdocs"
DEPOT_VERSIONS_DIRNAME = ["versions", "0"]
+DEPOT_STATUS_DIRNAME = ["status", "0"]
DEPOT_PUB_DIRNAME = ["publisher", "1"]
DEPOT_CACHE_FILENAME = "depot.cache"
@@ -87,6 +89,7 @@
catalog 1
file 1
manifest 0
+status 0
""" % pkg.VERSION
# versions response used when we provide search capability
@@ -177,7 +180,7 @@
default_pub = repository.cfg.get_property("publisher", "prefix")
except cfg.UnknownPropertyError:
default_pub = None
- return all_pubs, default_pub
+ return all_pubs, default_pub, repository.get_status()
def _write_httpd_conf(pubs, default_pubs, runtime_dir, log_dir, template_dir,
cache_dir, cache_size, host, port, sroot,
@@ -185,7 +188,8 @@
"""Writes the webserver configuration for the depot.
pubs repository and publisher information, a list in the form
- [(publisher_prefix, repo_dir, repo_prefix), ... ]
+ [(publisher_prefix, repo_dir, repo_prefix,
+ writable_root), ... ]
default_pubs default publishers, per repository, a list in the form
[(default_publisher_prefix, repo_dir, repo_prefix) ... ]
@@ -356,6 +360,19 @@
raise DepotException(
_("Unable to write publisher response: %s") % err)
+def _write_status_response(status, htdocs_path, repo_prefix):
+ """Writes a status status/0 response for the depot."""
+ try:
+ status_path = os.path.join(htdocs_path, repo_prefix,
+ os.path.sep.join(DEPOT_STATUS_DIRNAME), "index.html")
+ misc.makedirs(os.path.dirname(status_path))
+ with file(status_path, "w") as status_file:
+ status_file.write(json.dumps(status, ensure_ascii=False,
+ indent=2, sort_keys=True))
+ except OSError, err:
+ raise DepotException(
+ _("Unable to write status response: %s") % err)
+
def cleanup_htdocs(htdocs_dir):
"""Destroy any existing "htdocs" directory."""
try:
@@ -380,34 +397,40 @@
misc.makedirs(htdocs_path)
# pubs and default_pubs are lists of tuples of the form:
- # (publisher prefix, repository root dir, repository prefix)
+ # (publisher prefix, repository root dir, repository prefix,
+ # writable_root)
pubs = []
default_pubs = []
-
- repo_prefixes = [prefix for root, prefix in repo_info]
errors = []
# Query each repository for its publisher information.
- for (repo_root, repo_prefix) in repo_info:
+ for (repo_root, repo_prefix, writable_root) in repo_info:
try:
- publishers, default_pub = \
+ publishers, default_pub, status = \
_get_publishers(repo_root)
for pub in publishers:
pubs.append(
(pub, repo_root,
- repo_prefix))
+ repo_prefix, writable_root))
default_pubs.append((default_pub,
repo_root, repo_prefix))
+ _write_status_response(status, htdocs_path,
+ repo_prefix)
+ # The writable root must exist and must be
+ # owned by pkg5srv:pkg5srv
+ if writable_root:
+ misc.makedirs(writable_root)
+ _chown_dir(writable_root)
except DepotException, err:
errors.append(str(err))
if errors:
- raise DepotException(_("Unable to get publisher "
- "information: %s") % "\n".join(errors))
+ raise DepotException(_("Unable to write configuration: "
+ "%s") % "\n".join(errors))
# Write the publisher/0 response for each repository
pubs_by_repo = {}
- for pub_prefix, repo_root, repo_prefix in pubs:
+ for pub_prefix, repo_root, repo_prefix, writable_root in pubs:
pubs_by_repo.setdefault(repo_prefix, []).append(
pub_prefix)
for repo_prefix in pubs_by_repo:
@@ -439,6 +462,9 @@
for fmri in smf_instances:
repo_prefix = fmri.split(":")[-1]
repo_root = smf.get_prop(fmri, "pkg/inst_root")
+ writable_root = smf.get_prop(fmri, "pkg/writable_root")
+ if not writable_root or writable_root == '""':
+ writable_root = None
state = smf.get_prop(fmri, "restarter/state")
readonly = smf.get_prop(fmri, "pkg/readonly")
standalone = smf.get_prop(fmri, "pkg/standalone")
@@ -447,7 +473,7 @@
readonly == "true" and
standalone == "false"):
repo_info.append((repo_root,
- _affix_slash(repo_prefix)))
+ _affix_slash(repo_prefix), writable_root))
if not repo_info:
raise DepotException(_(
"No online, readonly, non-standalone instances of "
@@ -462,14 +488,22 @@
prefixes = set()
roots = set()
+ writable_roots = set()
errors = []
- for root, prefix in repo_info:
+ for root, prefix, writable_root in repo_info:
if prefix in prefixes:
- errors.append(_("instance %s already exists") % prefix)
+ errors.append(_("prefix %s cannot be used more than "
+ "once in a given depot configuration") % prefix)
prefixes.add(prefix)
if root in roots:
- errors.append(_("repo_root %s already exists") % root)
+ errors.append(_("repo_root %s cannot be used more "
+ "than once in a given depot configuration") % root)
roots.add(root)
+ if writable_root and writable_root in writable_roots:
+ errors.append(_("writable_root %s cannot be used more "
+ "than once in a given depot configuration") %
+ writable_root)
+ writable_roots.add(writable_root)
if errors:
raise DepotException("\n".join(errors))
return True
@@ -492,7 +526,7 @@
port = None
# a list of (repo_dir, repo_prefix) tuples
repo_info = []
- # the path where we store indexes and disk caches
+ # the path where we store disk caches
cache_dir = None
# our maximum cache size, in megabytes
cache_size = 0
@@ -517,6 +551,8 @@
# the current server_type
server_type = "apache2"
+ writable_root_set = False
+
try:
opts, pargs = getopt.getopt(sys.argv[1:],
"Ac:d:Fh:l:P:p:r:Ss:t:T:?", ["help", "debug="])
@@ -542,9 +578,17 @@
elif opt == "-d":
if "=" not in arg:
usage(_("-d arguments must be in the "
- "form <prefix>=<repo path>"))
- prefix, root = arg.split("=", 1)
- repo_info.append((root, _affix_slash(prefix)))
+ "form <prefix>=<repo path>"
+ "[=writable root]"))
+ components = arg.split("=", 2)
+ if len(components) == 3:
+ prefix, root, writable_root = components
+ writable_root_set = True
+ elif len(components) == 2:
+ prefix, root = components
+ writable_root = None
+ repo_info.append((root, _affix_slash(prefix),
+ writable_root))
elif opt == "-P":
sroot = _affix_slash(arg)
elif opt == "-F":
@@ -571,7 +615,7 @@
if not runtime_dir:
usage(_("required runtime dir option -r missing."))
- # we need a cache_dir to store the search indexes
+ # we need a cache_dir to store the SSLSessionCache
if not cache_dir and not fragment:
usage(_("cache_dir option -c is required if -F is not used."))
@@ -585,6 +629,13 @@
if repo_info and use_smf_instances:
usage(_("cannot use -d and -S together."))
+ # We can't support httpd.conf fragments with writable root, because
+ # we don't have the mod_wsgi app that can build the index or serve
+ # search requests everywhere the fragments might be used. (eg. on
+ # non-Solaris systems)
+ if writable_root_set and fragment:
+ usage(_("cannot use -d with writable roots and -F together."))
+
if fragment and port:
usage(_("cannot use -F and -p together."))
@@ -608,7 +659,10 @@
{"type": server_type,
"known": ", ".join(KNOWN_SERVER_TYPES)})
- _check_unique_repo_properties(repo_info)
+ try:
+ _check_unique_repo_properties(repo_info)
+ except DepotException, e:
+ error(e)
ret = refresh_conf(repo_info, log_dir, host, port, runtime_dir,
template_dir, cache_dir, cache_size, sroot, fragment=fragment,
--- a/src/tests/cli/t_depot_config.py Tue Nov 26 11:03:16 2013 +1300
+++ b/src/tests/cli/t_depot_config.py Wed Dec 04 15:55:52 2013 +1300
@@ -59,6 +59,8 @@
# FMRI STATE
["svc:/application/pkg/server:default", "online" ],
["svc:/application/pkg/server:usr", "online" ],
+ # an instance which we have a writable_root for
+ ["svc:/application/pkg/server:windex", "online" ],
# repositories that we will not serve
["svc:/application/pkg/server:off", "offline"],
["svc:/application/pkg/server:writable", "online" ],
@@ -70,15 +72,16 @@
# must be in the same order as svcs_conf and the rows
# must correspond.
default_svcprop_conf = [
- # inst_root readonly standalone
- ["%(rdir1)s", "true", "false" ],
- ["%(rdir2)s", "true", "false" ],
+ # inst_root readonly standalone writable_root
+ ["%(rdir1)s", "true", "false", "\"\""],
+ ["%(rdir2)s", "true", "false", "\"\""],
+ ["%(rdir3)s", "true", "false", "%(index_dir)s"],
# we intentionally use non-existent repository
# paths in these services, and check they aren't
# present in the httpd.conf later.
- ["/pkg5/there/aint", "true", "false", "offline"],
- ["/pkg5/nobody/here", "false", "false" ],
- ["/pkg5/but/us/chickens", "true", "true" ],
+ ["/pkg5/there/aint", "true", "false", "\"\""],
+ ["/pkg5/nobody/here", "false", "false", "\"\""],
+ ["/pkg5/but/us/chickens", "true", "true", "\"\""],
]
sample_pkg = """
@@ -110,10 +113,14 @@
def setUp(self):
self.sc = None
- pkg5unittest.ApacheDepotTestCase.setUp(self, ["test1", "test2"])
+ pkg5unittest.ApacheDepotTestCase.setUp(self, ["test1", "test2",
+ "test3"])
self.rdir1 = self.dcs[1].get_repodir()
self.rdir2 = self.dcs[2].get_repodir()
+ self.rdir3 = self.dcs[3].get_repodir()
+ self.index_dir = os.path.join(self.test_root,
+ "depot_writable_root")
self.default_depot_runtime = os.path.join(self.test_root,
"depot_runtime")
self.default_depot_conf = os.path.join(
@@ -157,7 +164,8 @@
_svcprop_conf[index].insert(0, fmri)
_svcprop_conf[index].insert(1, state)
- rdirs = {"rdir1": self.rdir1, "rdir2": self.rdir2}
+ rdirs = {"rdir1": self.rdir1, "rdir2": self.rdir2,
+ "rdir3": self.rdir3, "index_dir": self.index_dir}
# construct two strings we can use as parameters to our
# __svc*_template values
@@ -197,6 +205,8 @@
# the httpd.conf should reference our repositories
self.file_contains(self.ac.conf, self.rdir1)
self.file_contains(self.ac.conf, self.rdir2)
+ self.file_contains(self.ac.conf, self.rdir3)
+ self.file_contains(self.ac.conf, self.index_dir)
# it should not reference the repositories that we have
# marked as offline, writable or standalone
self.file_doesnt_contain(self.ac.conf, "/pkg5/there/aint")
@@ -240,6 +250,16 @@
self.assert_("/tmp" in err, "error message did not contain "
"/tmp")
+ # ensure we pick up invalid writable_root directories
+ ret, output, err = self.depotconfig("-d blah=%s=/dev/null" %
+ self.rdir1, out=True, stderr=True, exit=1)
+
+ # but check that we allow valid writeable_roots
+ ret, output, err = self.depotconfig("-d blah=%s=%s" %
+ (self.rdir1, self.index_dir), out=True, stderr=True)
+ self.file_contains(self.default_depot_conf,
+ "PKG5_WRITABLE_ROOT_blah %s" % self.index_dir)
+
def test_3_invalid_htcache_dir(self):
"""We return an error given an invalid cache_dir"""
@@ -386,9 +406,9 @@
self.depotconfig("", exit=1)
# test that when we break one of the repositories we're
- # serving, that the remaining repositories are still accessible
+ # serving, the remaining repositories are still accessible
# from the bui. We need to fix the repo dir before rebuilding
- # the configuration, then break it once the depot has started
+ # the configuration, then break it once the depot has started.
os.rename(broken_rdir, self.rdir2)
self.depotconfig("")
os.rename(self.rdir2, broken_rdir)
@@ -437,6 +457,7 @@
self.pkgsend_bulk(rurl, self.sample_pkg)
self.pkgsend_bulk(rurl, self.sample_pkg_11)
self.pkgsend_bulk(self.dcs[2].get_repo_url(), self.new_pkg)
+ self.pkgrepo("-s %s refresh" % self.dcs[2].get_repo_url())
self.depotconfig("")
self.image_create()
@@ -525,17 +546,32 @@
def test_14_htpkgrepo(self):
"""Test that only the 'pkgrepo refresh' command works with the
- depot-config only when the -A flag is enabled. Test that
- the index does indeed get updated when a refresh is performed
- and that new package contents are visible."""
+ depot-config only when the -A flag is enabled and only on
+ the repository that has a writable root. Test that the index
+ does indeed get updated when a refresh is performed and that
+ new package contents are visible."""
rurl = self.dcs[1].get_repo_url()
+ nosearch_rurl = self.dcs[2].get_repo_url()
+ writable_rurl = self.dcs[3].get_repo_url()
+
self.pkgsend_bulk(rurl, self.sample_pkg)
- # allow index refreshes
+ # we have a search index on rurl
+ self.pkgrepo("-s %s refresh" % rurl)
+ self.pkgsend_bulk(writable_rurl, self.sample_pkg)
+
+ # we have no search index on nosearch_rurl
+ self.pkgsend_bulk(nosearch_rurl, self.sample_pkg)
+
+ # allow index refreshes for repositories that support them
+ # (ie. have a writable root)
self.depotconfig("-A")
self.start_depot()
self.image_create()
+
depot_url = "%s/default" % self.ac.url
+ windex_url = "%s/windex" % self.ac.url
+ nosearch_url = "%s/usr" % self.ac.url
# verify that list commands work
ret, output = self.pkgrepo("-s %s list -F tsv" % depot_url,
@@ -550,40 +586,54 @@
self.pkgrepo("-s %s set -p test1 foo/bar=baz" % depot_url,
exit=2)
+ # verify that status works
+ self.pkgrepo("-s %s info" % depot_url)
+ self.assert_("test1 1 online" in self.reduceSpaces(self.output))
+
# verify search works for packages in the repository
self.pkg("set-publisher -p %s" % depot_url)
self.pkg("search -s %s msgsh" % "%s" % depot_url,
exit=1)
self.pkg("search -s %s /usr/bin/sample" % depot_url)
+ # Can't refresh this repo since it doesn't have a writable root
+ self.pkgrepo("-s %s refresh" % depot_url, exit=1)
+
+ # verify that search fails for repositories that don't have
+ # a pre-existing search index in the repository
+ self.pkg("search -s %s /usr/bin/sample" % nosearch_url, exit=1)
+
# publish a new package, and verify it doesn't appear in the
- # search results
- self.pkgsend_bulk(rurl, self.new_pkg)
- self.pkg("search -s %s /usr/bin/new" % depot_url, exit=1)
+ # search results for the repo with the writable_root
+ self.pkgsend_bulk(writable_rurl, self.new_pkg)
+ self.pkg("search -s %s /usr/bin/new" % windex_url, exit=1)
- # refresh the index
- self.pkgrepo("-s %s refresh" % depot_url)
+ # now refresh the index
+ self.pkgrepo("-s %s refresh" % windex_url)
+
# there isn't a synchronous option to pkgrepo, so wait a bit
# then make sure we do see this new package.
time.sleep(3)
- ret, output = self.pkg("search -s %s /usr/bin/new" % depot_url,
+
+ # we should now get search results for that new package
+ ret, output = self.pkg("search -s %s /usr/bin/new" % windex_url,
out=True)
self.assert_("usr/bin/new" in output)
- ret, output = self.pkgrepo("-s %s list -F tsv" % depot_url,
+ ret, output = self.pkgrepo("-s %s list -F tsv" % windex_url,
out=True)
- self.assert_("pkg://test1/[email protected]" in output)
- self.assert_("pkg://test1/[email protected]" in output)
+ self.assert_("pkg://test3/[email protected]" in output)
+ self.assert_("pkg://test3/[email protected]" in output)
# ensure that refresh --no-catalog works, but refresh --no-index
# does not.
- self.pkgrepo("-s %s refresh --no-catalog" % depot_url)
- self.pkgrepo("-s %s refresh --no-index" % depot_url, exit=1)
+ self.pkgrepo("-s %s refresh --no-catalog" % windex_url)
+ self.pkgrepo("-s %s refresh --no-index" % windex_url, exit=1)
# check that when we start the depot without -A, we cannot
# issue refresh commands.
self.depotconfig("")
self.start_depot()
- self.pkgrepo("-s %s refresh" % depot_url, exit=1)
+ self.pkgrepo("-s %s refresh" % windex_url, exit=1)
def test_15_htheaders(self):
"""Test that the correct Content-Type and Cache-control headers
@@ -591,15 +641,20 @@
fmris = self.pkgsend_bulk(self.dcs[1].get_repo_url(),
self.sample_pkg)
+ self.pkgrepo("-s %s refresh" % self.dcs[1].get_repo_url())
self.depotconfig("")
self.start_depot()
- # create an image so we have something to search with
- # (bug 15807844) then retrieve the hash of a file we have
- # published
+
+ # Create an image so we have something to search with.
+ # This technically isn't necessary anymore, but the test suite
+ # runs with some debug flags to make it (intentionally)
+ # difficult to mess with the root image of the test system
+ # (even though calling 'pkg search -s' would never actually
+ # modify it) Creating an image is just the easier thing to do
+ # here.
self.image_create()
- self.pkg("set-publisher -p %s/default" % self.ac.url)
- ret, output = self.pkg("search -H -o action.hash "
- "-r /usr/bin/sample", out=True)
+ ret, output = self.pkg("search -s %s/default -H -o action.hash "
+ "-r /usr/bin/sample" % self.ac.url, out=True)
file_hash = output.strip()
fmri = pkg.fmri.PkgFmri(fmris[0])
@@ -663,9 +718,18 @@
self.carrots_pkg)
self.pkgsend_bulk(self.dcs[2].get_repo_url(),
self.new_pkg)
+
+ # We shouldn't be able to supply a writable root when running
+ # in fragment mode
+ self.depotconfig("-l %s -F -d usr=%s -d spaghetti=%s=%s "
+ "-P testpkg5" %
+ (self.default_depot_runtime, self.rdir1, self.rdir2,
+ self.index_dir), exit=2)
+
self.depotconfig("-l %s -F -d usr=%s -d spaghetti=%s "
"-P testpkg5" %
(self.default_depot_runtime, self.rdir1, self.rdir2))
+
default_httpd_conf_path = os.path.join(self.test_root,
"default_httpd.conf")
httpd_conf = open(default_httpd_conf_path, "w")
@@ -755,10 +819,10 @@
#
# This script produces false svcprop(1) output, using
# a list of space separated strings, with each string
-# of the format <fmri>%%<state>%%<inst_root>%%<readonly>%%<standalone>
+# of the format <fmri>%%<state>%%<inst_root>%%<readonly>%%<standalone>%%<writable_root>
#
# eg.
-# SERVICE_PROPS="svc:/application/pkg/server:foo%%online%%/space/repo%%true%%false"
+# SERVICE_PROPS="svc:/application/pkg/server:foo%%online%%/space/repo%%true%%false%%/space/writable_root"
#
# we expect to be called as "svcprop -c -p <property> <fmri>"
# which is enough svcprop(1) functionalty for these tests. Any other
@@ -769,17 +833,19 @@
typeset -A prop_readonly
typeset -A prop_inst_root
typeset -A prop_standalone
+typeset -A prop_writable_root
SERVICE_PROPS="%s"
for service in $SERVICE_PROPS ; do
echo $service | sed -e 's/%%/ /g' | \
- read fmri state inst_root readonly standalone
+ read fmri state inst_root readonly standalone writable_root
# create a hashable version of the FMRI
fmri=$(echo $fmri | sed -e 's/\///g' -e 's/://g')
prop_state[$fmri]=$state
prop_inst_root[$fmri]=$inst_root
prop_readonly[$fmri]=$readonly
prop_standalone[$fmri]=$standalone
+ prop_writable_root[$fmri]=$writable_root
done
@@ -794,6 +860,9 @@
"pkg/standalone")
echo ${prop_standalone[$FMRI]}
;;
+ "pkg/writable_root")
+ echo ${prop_writable_root[$FMRI]}
+ ;;
"restarter/state")
echo ${prop_state[$FMRI]}
;;
--- a/src/util/apache2/depot/depot.conf.mako Tue Nov 26 11:03:16 2013 +1300
+++ b/src/util/apache2/depot/depot.conf.mako Wed Dec 04 15:55:52 2013 +1300
@@ -56,9 +56,9 @@
root = context.get("sroot")
runtime_dir = context.get("runtime_dir")
-for pub, repo_path, repo_prefix in pubs:
+for pub, repo_path, repo_prefix, writable_root in pubs:
repo_prefixes.add(repo_prefix)
-context.write("# per-repository versions and publishers responses\n")
+context.write("# per-repository versions, publishers and status responses\n")
for repo_prefix in repo_prefixes:
context.write(
@@ -67,6 +67,9 @@
context.write("RewriteRule ^/%(root)s%(repo_prefix)spublisher/0 "
"/%(root)s%(repo_prefix)spublisher/1/index.html [PT,NE]\n" %
locals())
+ context.write(
+ "RewriteRule ^/%(root)s%(repo_prefix)sstatus/0 "
+ "/%(root)s%(repo_prefix)sstatus/0/index.html [PT,NE]\n" % locals())
%>
<%doc>
@@ -116,7 +119,7 @@
%>
# Write per-publisher rules for publisher, version, file and manifest responses
-% for pub, repo_path, repo_prefix in pubs:
+% for pub, repo_path, repo_prefix, writable_root in pubs:
<%doc>
# Point to our local versions/0 response or
# publisher-specific publisher/1, response, then stop.
@@ -196,7 +199,7 @@
<%
paths = set()
root = context.get("sroot")
-for pub, repo_path, repo_prefix in pubs:
+for pub, repo_path, repo_prefix, writable_root in pubs:
paths.add((repo_path, repo_prefix))
context.write(
"Alias /%(root)s%(repo_prefix)s%(pub)s %(repo_path)s\n" %
--- a/src/util/apache2/depot/depot_httpd.conf.mako Tue Nov 26 11:03:16 2013 +1300
+++ b/src/util/apache2/depot/depot_httpd.conf.mako Wed Dec 04 15:55:52 2013 +1300
@@ -115,7 +115,6 @@
LimitRequestBody 102400
# Set environment variables used by our wsgi application
SetEnv PKG5_RUNTIME_DIR ${runtime_dir}
-SetEnv PKG5_CACHE_DIR ${cache_dir}
#
# If you wish httpd to run as a different user or group, you must run
@@ -345,12 +344,17 @@
path_info = set()
root = context.get("sroot")
context.write("# the repositories our search app should index.\n")
- for pub, repo_path, repo_prefix in pubs:
- path_info.add((repo_path, repo_prefix.rstrip("/")))
- for repo_path, repo_prefix in path_info:
+ for pub, repo_path, repo_prefix, writable_root in pubs:
+ path_info.add(
+ (repo_path, repo_prefix.rstrip("/"), writable_root))
+ for repo_path, repo_prefix, writable_root in path_info:
context.write(
"SetEnv PKG5_REPOSITORY_%(repo_prefix)s %(repo_path)s\n" %
locals())
+ if writable_root:
+ context.write(
+ "SetEnv PKG5_WRITABLE_ROOT_%(repo_prefix)s "
+ "%(writable_root)s\n" % locals())
context.write("RewriteRule ^/%(root)s%(repo_prefix)s/[/]?$ "
"%(root)s/depot/%(repo_prefix)s/ [NE,PT]\n" %
locals())
@@ -358,7 +362,7 @@
"%(root)s/depot/%(repo_prefix)s/$1 [NE,PT]\n" %
locals())
%>
-% for pub, repo_path, repo_prefix in pubs:
+% for pub, repo_path, repo_prefix, writable_root in pubs:
% if int(cache_size) > 0:
CacheEnable disk /${root}${repo_prefix}${pub}/file
CacheEnable disk /${root}${repo_prefix}${pub}/manifest
--- a/src/util/apache2/depot/depot_index.py Tue Nov 26 11:03:16 2013 +1300
+++ b/src/util/apache2/depot/depot_index.py Wed Dec 04 15:55:52 2013 +1300
@@ -21,20 +21,22 @@
#
# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+import atexit
import cherrypy
import httplib
import logging
import mako
import os
import re
+import shutil
import sys
+import tempfile
import threading
import time
import traceback
import urllib
import Queue
-import pkg.digest as digest
import pkg.p5i
import pkg.server.api
import pkg.server.repository as sr
@@ -44,14 +46,18 @@
# redirecting stdout for proper WSGI portability
sys.stdout = sys.stderr
-# a global dictionary containing lists of sr.Repository objects, keyed by
+# a global dictionary containing sr.Repository objects, keyed by
# repository prefix (not publisher prefix).
repositories = {}
-# a global dictionary containing lists of DepotBUI objects, keyed by repository
+# a global dictionary containing DepotBUI objects, keyed by repository
# prefix.
depot_buis = {}
+# a global dictionary containing sd.DepotHTTP objects, keyed by repository
+# prefix
+depot_https = {}
+
# a lock used during server startup to ensure we don't try to index the same
# repository at once.
repository_lock = threading.Lock()
@@ -59,10 +65,10 @@
import gettext
gettext.install("/")
-# How often we ping the depot while long-running background tasks are running
-# this should be set to less than the mod_wsgi inactivity-timeout (since
+# How often we ping the depot while long-running background tasks are running.
+# This should be set to less than the mod_wsgi inactivity-timeout (since
# pinging the depot causes activity, preventing mod_wsgi from shutting down the
-# Python interpreter.
+# Python interpreter.)
KEEP_BUSY_INTERVAL = 120
class DepotException(Exception):
@@ -75,6 +81,7 @@
def __str__(self):
return "%s: %s" % (self.message, self.request)
+
class AdminOpsDisabledException(DepotException):
"""An exception thrown when this wsgi application hasn't been configured
to allow admin/0 pkg(5) depot responses."""
@@ -105,6 +112,22 @@
"type": self.cmd}
+class IndexOpDisabledException(DepotException):
+ """An exception thrown when we've been asked to refresh an index for
+ a repository that doesn't have a writable_root property set."""
+
+ def __init__(self, request):
+ self.request = request
+ self.http_status = httplib.FORBIDDEN
+
+ def __str__(self):
+ return "admin/0 operations to refresh indexes are not " \
+ "allowed on this repository because it is read-only and " \
+ "the svc:/application/pkg/server instance does not have " \
+ "a config/writable_root SMF property set. " \
+ "Request was: %s" % self.request
+
+
class BackgroundTask(object):
"""Allow us to process a limited set of threads in the background."""
@@ -199,10 +222,17 @@
web resources (css, html, etc)
"""
- def __init__(self, repo, dconf, tmp_root, pkg5_test_proto=""):
+ def __init__(self, repo, dconf, writable_root, pkg5_test_proto=""):
self.repo = repo
self.cfg = dconf
- self.tmp_root = tmp_root
+ if writable_root:
+ self.tmp_root = writable_root
+ else:
+ self.tmp_root = tempfile.mkdtemp(prefix="pkg-depot.")
+ # try to clean up the temporary area on exit
+ atexit.register(shutil.rmtree, self.tmp_root,
+ ignore_errors=True)
+
# we hardcode these for the depot.
self.content_root = "%s/usr/share/lib/pkg" % pkg5_test_proto
self.web_root = "%s/usr/share/lib/pkg/web/" % pkg5_test_proto
@@ -211,6 +241,7 @@
# creating DepotHTTP objects.
self.cfg.set_property("pkg", "content_root", self.content_root)
self.cfg.set_property("pkg", "pkg_root", self.repo.root)
+ self.cfg.set_property("pkg", "writable_root", self.tmp_root)
face.init(self)
@@ -226,15 +257,18 @@
PKG5_RUNTIME_DIR A directory that contains runtime state, notably
a htdocs/ directory.
- PKG5_CACHE_DIR Where we can store search indices for the repositories
- we're managing.
+ PKG5_REPOSITORY_<repo_prefix> A path to the repository root for the
+ given <repo_prefix>. <repo_prefix> is a unique
+ alphanumeric prefix for the depot, each corresponding
+ to a given <repo_root>. Many PKG5_REPOSITORY_*
+ variables can be configured, possibly containing
+ pkg5 publishers of the same name.
- PKG5_REPOSITORY_* A colon-separated pair of values, in the form
- <repo_prefix>:<repo_root>. <repo_prefix> is a unique
- alphanumeric prefix that maps to the given
- <repo_root>. Many PKG5_REPOSITORY_* variables can be
- configured, possibly containing identical pkg5
- publishers.
+ PKG5_WRITABLE_ROOT_<repo_prefix> A path to the writable root for the
+ given <repo_prefix>. If a writable root is not set,
+ and a search index does not already exist in the
+ repository root, search functionality is not
+ available.
PKG5_ALLOW_REFRESH Set to 'true', this determines whether we process
admin/0 requests that have the query 'cmd=refresh' or
@@ -254,7 +288,6 @@
"""
def __init__(self):
- self.cache_dir = None
self.bgtask = None
def setup(self, request):
@@ -262,6 +295,9 @@
repository prefix, and ensures our search indexes exist."""
def get_repo_paths():
+ """Return a dictionary of repository paths, and writable
+ root directories for the repositories we're serving."""
+
repo_paths = {}
for key in request.wsgi_environ:
if key.startswith("PKG5_REPOSITORY"):
@@ -269,6 +305,12 @@
"")
repo_paths[prefix] = \
request.wsgi_environ[key]
+ writable_root = \
+ request.wsgi_environ.get(
+ "PKG5_WRITABLE_ROOT_%s" % prefix)
+ repo_paths[prefix] = \
+ (request.wsgi_environ[key],
+ writable_root)
return repo_paths
if repositories:
@@ -280,8 +322,6 @@
repository_lock.acquire()
repo_paths = get_repo_paths()
- self.cache_dir = request.wsgi_environ.get("PKG5_CACHE_DIR",
- "/var/cache/pkg/depot")
# We must ensure our BackgroundTask object has at least as many
# slots available as we have repositories, to allow the indexes
@@ -294,28 +334,20 @@
self.bgtask.start()
for prefix in repo_paths:
- path = repo_paths[prefix]
- repo_hash = digest.DEFAULT_HASH_FUNC(path).hexdigest()
- index_dir = os.path.sep.join(
- [self.cache_dir, "indexes", repo_hash])
-
- # if the index dir exists for this repository, we do not
- # automatically attempt a refresh.
- refresh_index = not os.path.exists(index_dir)
+ path, writable_root = repo_paths[prefix]
try:
- repo = sr.Repository(root=path,
- read_only=True, writable_root=index_dir)
+ repo = sr.Repository(root=path, read_only=True,
+ writable_root=writable_root)
except sr.RepositoryError, e:
- print("Error initializing repository at %s: "
- "%s" % (path, e))
+ print "Unable to load repository: %s" % e
continue
repositories[prefix] = repo
dconf = sd.DepotConfig()
- if refresh_index:
+ if writable_root is not None:
self.bgtask.put(repo.refresh_index)
- depot = DepotBUI(repo, dconf, index_dir,
+ depot = DepotBUI(repo, dconf, writable_root,
pkg5_test_proto=pkg5_test_proto)
depot_buis[prefix] = depot
@@ -493,6 +525,9 @@
repo = repositories[repo_prefix]
depot_bui = depot_buis[repo_prefix]
+ if repo_prefix in depot_https:
+ return depot_https[repo_prefix]
+
def request_pub_func(path_info):
"""A function that can be called to determine the
publisher for a given request. We always want None
@@ -502,8 +537,9 @@
"""
return None
- return sd.DepotHTTP(repo, depot_bui.cfg,
+ depot_https[repo_prefix] = sd.DepotHTTP(repo, depot_bui.cfg,
request_pub_func=request_pub_func)
+ return depot_https[repo_prefix]
def __strip_pub(self, tokens, repo):
"""Attempt to strip at most one publisher from the path
@@ -532,7 +568,7 @@
return dh.info_0(*tokens[3:])
def p5i(self, *tokens):
- """Use a into DepotHTTP to return a p5i response."""
+ """Use a DepotHTTP to return a p5i response."""
dh = self.__build_depot_http()
tokens = self.__strip_pub(tokens, dh.repo)
@@ -597,6 +633,12 @@
if pub_prefix not in repo.publishers:
raise cherrypy.NotFound()
+ # Since the repository is read-only, we only honour
+ # index refresh requests if we have a writable root.
+ if not repo.writable_root:
+ raise IndexOpDisabledException(
+ request.wsgi_environ["REQUEST_URI"])
+
# we need to reload the repository in order to get
# any new catalog contents before refreshing the
# index.
@@ -746,6 +788,10 @@
elif isinstance(e, AdminOpNotSupportedException):
raise cherrypy.HTTPError(e.http_status,
"This operation is not supported.")
+ elif isinstance(e, IndexOpDisabledException):
+ raise cherrypy.HTTPError(e.http_status,
+ "This operation has been disabled by the "
+ "server administrator.")
else:
# we leave this as a 500 for now. It will be
# converted and logged by our error handler