s1890 - Allowed for the AC web service to be in READONLY and OFFLINE modes

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/ldap/LdapConnectionPool.java

@@ -100,6 +100,13 @@ public class LdapConnectionPool
 {
     private static final Logger logger = Logger.getLogger(LdapConnectionPool.class);
 
+    private enum SystemState
+    {
+        ONLINE,
+        READONLY,
+        OFFLINE
+    };
+
     Profiler profiler = new Profiler(LdapConnectionPool.class);
 
     protected LdapConfig currentConfig;
@@ -107,8 +114,10 @@ public class LdapConnectionPool
     private LDAPConnectionPool pool;
     private Object poolMonitor = new Object();
     private LDAPConnectionOptions connectionOptions;
+    private boolean readOnly;
+    SystemState systemState = SystemState.ONLINE;
 
-    public LdapConnectionPool(LdapConfig config, LdapPool poolConfig, String poolName, boolean boundPool)
+    public LdapConnectionPool(LdapConfig config, LdapPool poolConfig, String poolName, boolean boundPool, boolean readOnly)
     {
         if (config == null)
             throw new IllegalArgumentException("config required");
@@ -122,19 +131,49 @@ public class LdapConnectionPool
         connectionOptions.setAutoReconnect(true);
         currentConfig = config;
         this.poolName = poolName;
+        this.readOnly = readOnly;
+
+        systemState = getSystemState(config);
+        logger.debug("Construct pool: " + poolName + ". system state: " + systemState);
+        if (SystemState.ONLINE.equals(systemState) || (SystemState.READONLY.equals(systemState) && readOnly))
+        {
             synchronized (poolMonitor)
             {
                 if (!boundPool)
                     pool = createPool(config, poolConfig, poolName, null, null);
                 else
                     pool = createPool(config, poolConfig, poolName, config.getAdminUserDN(), config.getAdminPasswd());
+
+                if (pool != null)
+                {
                     logger.debug(poolName + " statistics after create:\n" + pool.getConnectionPoolStatistics());
                     profiler.checkpoint("Create read only pool.");
                 }
             }
+        }
+        else
+        {
+            logger.debug("Not creating pool " + poolName + " because system state is " + systemState);
+        }
+    }
 
     public LDAPConnection getConnection() throws TransientException
     {
+
+        logger.debug("Get connection: " + poolName + ". system state: " + systemState);
+        if (SystemState.OFFLINE.equals(systemState))
+        {
+            throw new TransientException("The system is down for maintenance.", 600);
+        }
+
+        if (SystemState.READONLY.equals(systemState))
+        {
+            if (!readOnly)
+            {
+                throw new TransientException("The system is in read-only mode.", 600);
+            }
+        }
+
         try
         {
             LDAPConnection conn = null;
@@ -168,10 +207,13 @@ public class LdapConnectionPool
     }
 
     public void releaseConnection(LDAPConnection conn)
+    {
+        if (pool != null)
         {
             pool.releaseConnection(conn);
             logger.debug(poolName + " pool statistics after release:\n" + pool.getConnectionPoolStatistics());
         }
+    }
 
     public LdapConfig getCurrentConfig()
     {
@@ -179,11 +221,14 @@ public class LdapConnectionPool
     }
 
     public void shutdown()
+    {
+        if (pool != null)
         {
             logger.debug("Closing pool...");
             pool.close();
             profiler.checkpoint("Pool closed.");
         }
+    }
 
     public String getName()
     {
@@ -191,7 +236,6 @@ public class LdapConnectionPool
     }
 
     private LDAPConnectionPool createPool(LdapConfig config, LdapPool poolConfig, String poolName, String bindID, String bindPW)
-
     {
         try
         {
@@ -245,4 +289,32 @@ public class LdapConnectionPool
     }
 
 
+    /**
+     * Check if in read-only or offline mode.
+     *
+     * A read max connection size of zero implies offline mode.
+     * A read-wrtie max connection size of zero implies read-only mode.
+     */
+    private SystemState getSystemState(LdapConfig config)
+    {
+
+        if (config.getReadOnlyPool().getMaxSize() == 0)
+        {
+            return SystemState.OFFLINE;
+        }
+
+        if (config.getUnboundReadOnlyPool().getMaxSize() == 0)
+        {
+            return SystemState.OFFLINE;
+        }
+
+        if (config.getReadWritePool().getMaxSize() == 0)
+        {
+            return SystemState.READONLY;
+        }
+
+        return SystemState.ONLINE;
+    }
+
+
 }

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/ldap/LdapConnections.java

@@ -147,7 +147,7 @@ class LdapConnections
         {
             if (readOnlyPool == null)
             {
-                readOnlyPool = new LdapConnectionPool(config, config.getReadOnlyPool(), LdapPersistence.POOL_READONLY, true);
+                readOnlyPool = new LdapConnectionPool(config, config.getReadOnlyPool(), LdapPersistence.POOL_READONLY, true, true);
             }
             if (manualConfigReadOnlyConn == null)
             {
@@ -186,7 +186,7 @@ class LdapConnections
         {
             if (readWritePool == null)
             {
-                readWritePool = new LdapConnectionPool(config, config.getReadWritePool(), LdapPersistence.POOL_READWRITE, true);
+                readWritePool = new LdapConnectionPool(config, config.getReadWritePool(), LdapPersistence.POOL_READWRITE, true, false);
             }
             if (manualConfigReadWriteConn == null)
             {
@@ -225,7 +225,7 @@ class LdapConnections
         {
             if (unboundReadOnlyPool == null)
             {
-                unboundReadOnlyPool = new LdapConnectionPool(config, config.getUnboundReadOnlyPool(), LdapPersistence.POOL_UNBOUNDREADONLY, false);
+                unboundReadOnlyPool = new LdapConnectionPool(config, config.getUnboundReadOnlyPool(), LdapPersistence.POOL_UNBOUNDREADONLY, false, true);
             }
             if (manualConfigUnboundReadOnlyConn == null)
             {
@@ -317,4 +317,5 @@ class LdapConnections
             return config;
     }
 
+
 }

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/ldap/LdapGroupPersistence.java

@@ -70,7 +70,13 @@ package ca.nrc.cadc.ac.server.ldap;
 
 import java.security.AccessControlException;
 import java.security.Principal;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import javax.security.auth.Subject;
 
 import org.apache.log4j.Logger;
 
@@ -87,11 +93,6 @@ import ca.nrc.cadc.auth.AuthMethod;
 import ca.nrc.cadc.auth.AuthenticationUtil;
 import ca.nrc.cadc.auth.DNPrincipal;
 import ca.nrc.cadc.net.TransientException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import javax.security.auth.Subject;
 
 public class LdapGroupPersistence<T extends Principal> extends LdapPersistence implements GroupPersistence<T>
 {

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/ldap/LdapPersistence.java

@@ -115,7 +115,7 @@ public abstract class LdapPersistence
             ConnectionPools pools = lookupPools();
             if (pools == null || pools.isClosed())
                 throw new IllegalStateException("Pools are closed.");
-            poolCheck(pools);
+            pools = poolCheck(pools);
             return pools.getPools().get(poolName);
         }
         catch (NamingException e)
@@ -240,11 +240,11 @@ public abstract class LdapPersistence
     {
         Map<String,LdapConnectionPool> poolMap = new HashMap<String,LdapConnectionPool>(3);
         poolMap.put(POOL_READONLY, new LdapConnectionPool(
-            config, config.getReadOnlyPool(), POOL_READONLY, true));
+            config, config.getReadOnlyPool(), POOL_READONLY, true, true));
         poolMap.put(POOL_READWRITE, new LdapConnectionPool(
-            config, config.getReadWritePool(), POOL_READWRITE, true));
+            config, config.getReadWritePool(), POOL_READWRITE, true, false));
         poolMap.put(POOL_UNBOUNDREADONLY, new LdapConnectionPool(
-            config, config.getUnboundReadOnlyPool(), POOL_UNBOUNDREADONLY, false));
+            config, config.getUnboundReadOnlyPool(), POOL_UNBOUNDREADONLY, false, true));
         profiler.checkpoint("Created 3 LDAP connection pools");
         return new ConnectionPools(poolMap, config);
     }
@@ -262,7 +262,7 @@ public abstract class LdapPersistence
         }
     }
 
-    private void poolCheck(ConnectionPools pools) throws TransientException
+    private ConnectionPools poolCheck(ConnectionPools pools) throws TransientException
     {
         if (timeToCheckPools(pools))
         {
@@ -278,6 +278,7 @@ public abstract class LdapPersistence
                 logger.debug("Detected ldap configuration change, rebuilding pools");
                 boolean poolRecreated = false;
                 final ConnectionPools oldPools = pools;
+                ConnectionPools newPools = null;
 
                 synchronized (jndiMonitor)
                 {
@@ -287,7 +288,7 @@ public abstract class LdapPersistence
                     {
                         try
                         {
-                            ConnectionPools newPools = createPools(newConfig);
+                            newPools = createPools(newConfig);
                             newPools.setLastPoolCheck(System.currentTimeMillis());
                             InitialContext ic = new InitialContext();
                             try
@@ -326,9 +327,13 @@ public abstract class LdapPersistence
                     };
                     Thread closePoolsThread = new Thread(closeOldPools);
                     closePoolsThread.start();
+
+                    return newPools;
                 }
             }
         }
+        // just return the existing pools
+        return pools;
     }
 
     private boolean timeToCheckPools(ConnectionPools pools)

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/ldap/LdapUserPersistence.java

@@ -72,6 +72,8 @@ import java.security.AccessControlException;
 import java.security.Principal;
 import java.util.Collection;
 
+import javax.security.auth.Subject;
+
 import org.apache.log4j.Logger;
 
 import ca.nrc.cadc.ac.User;
@@ -85,8 +87,6 @@ import ca.nrc.cadc.auth.HttpPrincipal;
 import ca.nrc.cadc.net.TransientException;
 import ca.nrc.cadc.profiler.Profiler;
 
-import javax.security.auth.Subject;
-
 public class LdapUserPersistence<T extends Principal> extends LdapPersistence implements UserPersistence<T>
 {
     private static final Logger logger = Logger.getLogger(LdapUserPersistence.class);

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/web/ACSearchRunner.java

@@ -68,6 +68,24 @@
  */
 package ca.nrc.cadc.ac.server.web;
 
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.security.AccessControlContext;
+import java.security.AccessControlException;
+import java.security.AccessController;
+import java.security.Principal;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Set;
+
+import javax.security.auth.Subject;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.log4j.Logger;
+
 import ca.nrc.cadc.ac.Group;
 import ca.nrc.cadc.ac.GroupNotFoundException;
 import ca.nrc.cadc.ac.UserNotFoundException;
@@ -75,8 +93,6 @@ import ca.nrc.cadc.ac.server.GroupPersistence;
 import ca.nrc.cadc.ac.server.PluginFactory;
 import ca.nrc.cadc.ac.server.RequestValidator;
 import ca.nrc.cadc.ac.xml.GroupListWriter;
-import ca.nrc.cadc.auth.AuthenticationUtil;
-import ca.nrc.cadc.auth.HttpPrincipal;
 import ca.nrc.cadc.net.TransientException;
 import ca.nrc.cadc.uws.ExecutionPhase;
 import ca.nrc.cadc.uws.Job;
@@ -84,23 +100,6 @@ import ca.nrc.cadc.uws.server.JobRunner;
 import ca.nrc.cadc.uws.server.JobUpdater;
 import ca.nrc.cadc.uws.server.SyncOutput;
 import ca.nrc.cadc.uws.util.JobLogInfo;
-import org.apache.log4j.Logger;
-
-import javax.security.auth.Subject;
-import javax.security.auth.x500.X500Principal;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.security.AccessControlContext;
-import java.security.AccessControlException;
-import java.security.AccessController;
-import java.security.Principal;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.Set;
 
 public class ACSearchRunner implements JobRunner
 {
@@ -221,9 +220,11 @@ public class ACSearchRunner implements JobRunner
 
             syncOut.setResponseCode(503);
             syncOut.setHeader("Content-Type", "text/plain");
+            if (t.getRetryDelay() > 0)
+                syncOut.setHeader("Retry-After", Integer.toString(t.getRetryDelay()));
             try
             {
-                syncOut.getOutputStream().write(t.getMessage().getBytes());
+                syncOut.getOutputStream().write(("Transient Exception: " + t.getMessage()).getBytes());
             }
             catch (IOException e)
             {

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/web/LoginServlet.java

@@ -208,6 +208,18 @@ public class LoginServlet<T extends Principal> extends HttpServlet
             response.getWriter().write(message);
             response.setStatus(401);
         }
+        catch (TransientException e)
+        {
+            log.debug(e.getMessage(), e);
+            String message = e.getMessage();
+            logInfo.setMessage(message);
+            logInfo.setSuccess(false);
+            response.setContentType("CONTENT_TYPE");
+            if (e.getRetryDelay() > 0)
+                response.setHeader("Retry-After", Integer.toString(e.getRetryDelay()));
+            response.getWriter().write("Transient Error: " + message);
+            response.setStatus(503);
+        }
         catch (Throwable t)
         {
             String message = "Internal Server Error: " + t.getMessage();

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/web/ModifyPasswordServlet.java

@@ -68,13 +68,11 @@
  */
 package ca.nrc.cadc.ac.server.web;
 
-import ca.nrc.cadc.ac.server.PluginFactory;
-import ca.nrc.cadc.ac.server.UserPersistence;
-import ca.nrc.cadc.auth.AuthenticationUtil;
-import ca.nrc.cadc.auth.HttpPrincipal;
-import ca.nrc.cadc.log.ServletLogInfo;
-import ca.nrc.cadc.util.StringUtil;
-import org.apache.log4j.Logger;
+import java.io.IOException;
+import java.security.AccessControlException;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Set;
 
 import javax.security.auth.Subject;
 import javax.servlet.ServletConfig;
@@ -82,10 +80,16 @@ import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.security.AccessControlException;
-import java.security.PrivilegedExceptionAction;
-import java.util.Set;
+
+import org.apache.log4j.Logger;
+
+import ca.nrc.cadc.ac.server.PluginFactory;
+import ca.nrc.cadc.ac.server.UserPersistence;
+import ca.nrc.cadc.auth.AuthenticationUtil;
+import ca.nrc.cadc.auth.HttpPrincipal;
+import ca.nrc.cadc.log.ServletLogInfo;
+import ca.nrc.cadc.net.TransientException;
+import ca.nrc.cadc.util.StringUtil;
 
 /**
  * Servlet to handle password changes.  Passwords are an integral part of the
@@ -130,8 +134,7 @@ public class ModifyPasswordServlet extends HttpServlet
             logInfo.setSubject(subject);
             if ((subject == null) || (subject.getPrincipals().isEmpty()))
             {
-                logInfo.setMessage("Unauthorized subject");
-                response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+                throw new AccessControlException("Unauthorized");
             }
             else
             {
@@ -167,22 +170,52 @@ public class ModifyPasswordServlet extends HttpServlet
                 });
             }
         }
+        catch (Throwable t)
+        {
+            try
+            {
+                if (t instanceof PrivilegedActionException)
+                {
+                    Exception e = ((PrivilegedActionException) t).getException();
+                    if (e != null)
+                    {
+                        throw e;
+                    }
+                }
+
+                throw t;
+            }
             catch (IllegalArgumentException e)
             {
                 log.debug(e.getMessage(), e);
+                response.setContentType("text/plain");
                 logInfo.setMessage(e.getMessage());
                 response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
             }
             catch (AccessControlException e)
             {
                 log.debug(e.getMessage(), e);
+                response.setContentType("text/plain");
                 logInfo.setMessage(e.getMessage());
                 response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
             }
-        catch (Throwable t)
+            catch (TransientException e)
+            {
+                log.debug(e.getMessage(), e);
+                String message = e.getMessage();
+                logInfo.setMessage(message);
+                logInfo.setSuccess(false);
+                response.setContentType("text/plain");
+                if (e.getRetryDelay() > 0)
+                    response.setHeader("Retry-After", Integer.toString(e.getRetryDelay()));
+                response.getWriter().write("Transient Error: " + message);
+                response.setStatus(503);
+            }
+            catch (Throwable e)
             {
-            String message = "Internal Server Error: " + t.getMessage();
-            log.error(message, t);
+                String message = "Internal Server Error: " + e.getMessage();
+                log.error(message, e);
+                response.setContentType("text/plain");
                 logInfo.setSuccess(false);
                 logInfo.setMessage(message);
                 response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
@@ -193,6 +226,7 @@ public class ModifyPasswordServlet extends HttpServlet
                 log.info(logInfo.end());
             }
         }
+    }
 
     /**
      * Get and augment the Subject. Tests can override this method.

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/web/ResetPasswordServlet.java

@@ -68,29 +68,6 @@
  */
 package ca.nrc.cadc.ac.server.web;
 
-import ca.nrc.cadc.ac.User;
-import ca.nrc.cadc.ac.UserAlreadyExistsException;
-import ca.nrc.cadc.ac.UserNotFoundException;
-import ca.nrc.cadc.ac.server.ACScopeValidator;
-import ca.nrc.cadc.ac.server.PluginFactory;
-import ca.nrc.cadc.ac.server.UserPersistence;
-import ca.nrc.cadc.auth.AuthenticationUtil;
-import ca.nrc.cadc.auth.DelegationToken;
-import ca.nrc.cadc.auth.HttpPrincipal;
-import ca.nrc.cadc.auth.ServletPrincipalExtractor;
-import ca.nrc.cadc.log.ServletLogInfo;
-import ca.nrc.cadc.util.StringUtil;
-
-import org.apache.log4j.Logger;
-
-import javax.security.auth.Subject;
-import javax.security.auth.x500.X500Principal;
-import javax.servlet.ServletConfig;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
 import java.io.IOException;
 import java.net.URI;
 import java.security.AccessControlException;
@@ -99,12 +76,35 @@ import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Calendar;
-import java.util.Enumeration;
 import java.util.GregorianCalendar;
 import java.util.List;
 import java.util.Set;
 import java.util.TimeZone;
 
+import javax.security.auth.Subject;
+import javax.security.auth.x500.X500Principal;
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.log4j.Logger;
+
+import ca.nrc.cadc.ac.User;
+import ca.nrc.cadc.ac.UserAlreadyExistsException;
+import ca.nrc.cadc.ac.UserNotFoundException;
+import ca.nrc.cadc.ac.server.ACScopeValidator;
+import ca.nrc.cadc.ac.server.PluginFactory;
+import ca.nrc.cadc.ac.server.UserPersistence;
+import ca.nrc.cadc.auth.AuthenticationUtil;
+import ca.nrc.cadc.auth.DelegationToken;
+import ca.nrc.cadc.auth.HttpPrincipal;
+import ca.nrc.cadc.auth.ServletPrincipalExtractor;
+import ca.nrc.cadc.log.ServletLogInfo;
+import ca.nrc.cadc.net.TransientException;
+import ca.nrc.cadc.util.StringUtil;
+
 /**
  * Servlet to handle password resets.  Passwords are an integral part of the
  * access control system and are handled differently to accommodate stricter
@@ -308,6 +308,18 @@ public class ResetPasswordServlet extends HttpServlet
                 logInfo.setMessage(e.getMessage());
                 response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
             }
+            catch (TransientException e)
+            {
+                log.debug(e.getMessage(), e);
+                String message = e.getMessage();
+                logInfo.setMessage(message);
+                logInfo.setSuccess(false);
+                response.setContentType("text/plain");
+                if (e.getRetryDelay() > 0)
+                    response.setHeader("Retry-After", Integer.toString(e.getRetryDelay()));
+                response.getWriter().write("Transient Error: " + message);
+                response.setStatus(503);
+            }
             catch (AccessControlException e)
             {
                 log.debug(e.getMessage(), e);

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/web/groups/AbstractGroupAction.java

@@ -68,27 +68,24 @@
  */
 package ca.nrc.cadc.ac.server.web.groups;
 
+import java.io.IOException;
+import java.security.AccessControlException;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.log4j.Logger;
+
 import ca.nrc.cadc.ac.GroupAlreadyExistsException;
 import ca.nrc.cadc.ac.GroupNotFoundException;
 import ca.nrc.cadc.ac.MemberAlreadyExistsException;
 import ca.nrc.cadc.ac.MemberNotFoundException;
 import ca.nrc.cadc.ac.UserNotFoundException;
 import ca.nrc.cadc.ac.server.GroupPersistence;
-import ca.nrc.cadc.ac.server.PluginFactory;
-import ca.nrc.cadc.ac.server.UserPersistence;
 import ca.nrc.cadc.ac.server.web.SyncOutput;
 import ca.nrc.cadc.net.TransientException;
-import org.apache.log4j.Logger;
-
-import com.unboundid.ldap.sdk.LDAPException;
-
-import javax.servlet.http.HttpServletRequest;
-import java.io.IOException;
-import java.security.AccessControlException;
-import java.security.Principal;
-import java.security.PrivilegedActionException;
-import java.security.PrivilegedExceptionAction;
-import java.util.List;
 
 public abstract class AbstractGroupAction implements PrivilegedExceptionAction<Object>
 {
@@ -187,9 +184,11 @@ public abstract class AbstractGroupAction implements PrivilegedExceptionAction<O
         }
         catch (TransientException e)
         {
-            String message = "Internal Transient Error: " + e.getMessage();
+            String message = "Transient Error: " + e.getMessage();
             this.logInfo.setSuccess(false);
             this.logInfo.setMessage(message);
+            if (e.getRetryDelay() > 0)
+                syncOut.setHeader("Retry-After", Integer.toString(e.getRetryDelay()));
             log.error(message, e);
             sendError(503, message);
         }

cadcAccessControl-Server/src/ca/nrc/cadc/ac/server/web/users/AbstractUserAction.java

@@ -68,6 +68,16 @@
  */
 package ca.nrc.cadc.ac.server.web.users;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Writer;
+import java.security.AccessControlException;
+import java.security.Principal;
+import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
+
+import org.apache.log4j.Logger;
+
 import ca.nrc.cadc.ac.ReaderException;
 import ca.nrc.cadc.ac.User;
 import ca.nrc.cadc.ac.UserAlreadyExistsException;
@@ -86,17 +96,6 @@ import ca.nrc.cadc.ac.xml.UserWriter;
 import ca.nrc.cadc.net.TransientException;
 import ca.nrc.cadc.profiler.Profiler;
 
-import org.apache.log4j.Logger;
-
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Writer;
-import java.security.AccessControlException;
-import java.security.Principal;
-import java.security.PrivilegedExceptionAction;
-import java.util.Collection;
-
 public abstract class AbstractUserAction<T extends Principal> implements PrivilegedExceptionAction<Object>
 {
     private static final Logger log = Logger.getLogger(AbstractUserAction.class);
@@ -193,9 +192,11 @@ public abstract class AbstractUserAction<T extends Principal> implements Privile
         }
         catch (TransientException e)
         {
-            String message = "Internal Transient Error: " + e.getMessage();
+            String message = "Transient Error: " + e.getMessage();
             this.logInfo.setSuccess(false);
             this.logInfo.setMessage(message);
+            if (e.getRetryDelay() > 0)
+                syncOut.setHeader("Retry-After", Integer.toString(e.getRetryDelay()));
             log.error(message, e);
             sendError(503, message);
         }

cadcAccessControl-Server/test/src/ca/nrc/cadc/ac/server/ldap/LdapConnectionsTest.java

@@ -108,7 +108,7 @@ public class LdapConnectionsTest
             EasyMock.expect(persistence.getPool(LdapPersistence.POOL_READONLY)).andReturn(readPool).once();
             EasyMock.expect(persistence.getPool(LdapPersistence.POOL_READWRITE)).andReturn(writePool).once();
             EasyMock.expect(persistence.getPool(LdapPersistence.POOL_UNBOUNDREADONLY)).andReturn(unReadPool).once();
-            EasyMock.expect(persistence.getCurrentConfig()).andReturn(null).once();
+            EasyMock.expect(persistence.getCurrentConfig()).andReturn(null).anyTimes();
 
             EasyMock.expect(readPool.getConnection()).andReturn(readConn).once();
             EasyMock.expect(writePool.getConnection()).andReturn(writeConn).once();

cadcAccessControl-Server/test/src/ca/nrc/cadc/ac/server/web/ModifyPasswordServletTest.java

@@ -68,11 +68,13 @@
 
 package ca.nrc.cadc.ac.server.web;
 
-import ca.nrc.cadc.ac.server.UserPersistence;
-import ca.nrc.cadc.auth.HttpPrincipal;
-import ca.nrc.cadc.util.StringUtil;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
 
-import org.junit.Test;
+import java.security.PrivilegedExceptionAction;
 
 import javax.security.auth.Subject;
 import javax.servlet.ServletConfig;
@@ -80,14 +82,23 @@ import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import java.security.PrivilegedExceptionAction;
+import org.apache.log4j.Level;
+import org.junit.Test;
 
-import static org.easymock.EasyMock.*;
+import ca.nrc.cadc.ac.server.UserPersistence;
+import ca.nrc.cadc.auth.HttpPrincipal;
+import ca.nrc.cadc.util.Log4jInit;
+import ca.nrc.cadc.util.StringUtil;
 
 
 public class ModifyPasswordServletTest
 {
 
+    public ModifyPasswordServletTest()
+    {
+        Log4jInit.setLevel("ca.nrc.cadc.ac", Level.INFO);
+    }
+
     public void testSubjectAndPasswords(final Subject subject, final String oldPassword,
             final String newPassword, int responseStatus) throws Exception
     {
@@ -116,6 +127,12 @@ public class ModifyPasswordServletTest
             expect(mockRequest.getParameter("new_password")).andReturn(newPassword).once();
         }
 
+        if (responseStatus != 200)
+        {
+            mockResponse.setContentType("text/plain");
+            expectLastCall().once();
+        }
+
         mockResponse.setStatus(responseStatus);
         expectLastCall().once();
 
@@ -212,6 +229,9 @@ public class ModifyPasswordServletTest
 
         if (hasInternalServerError)
         {
+            mockResponse.setContentType("text/plain");
+            expectLastCall().once();
+
             mockResponse.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
             expectLastCall().once();
         }