// dialws.cxx

// Main program which provides the dial web service.

#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include <iomanip>
#include <sys/socket.h>
#include <signal.h>
#include <dlfcn.h>
#include "dataset_util/getcwd.h"
#include "dataset_util/get_hostname.h"
#include "dataset_util/PThreadMutex.h"
#include "dataset_util/PThreadCondition.h"
#include "dataset_util/pthread_sleep.h"
#include "dataset_util/FileStatus.h"
#include "dataset_util/Time.h"
#include "dataset_util/ThreadCredential.h"
#include "dial_gsi/FileGsiAuthorizer.h"
#include "dial_ws/dial_ws_flags.h"
#include "dataset_credential/GssCredentialManager.h"
#include "dataset_credential/CredentialSelectionCatalog.h"
#include "dataset_credential/Gsoap.h"
#include "dial_ws/GsoapRegistry.h"
#include "dial_ws/DialWs.h"
#include "gsoap/dial_soap.h"

#ifdef WITH_GSI
#include "gsi.h"
#endif

using std::string;
using std::cout;
using std::cerr;
using std::endl;
using std::ifstream;
using std::ostream;
using std::ofstream;
using std::istringstream;
using std::ostringstream;
using std::vector;
using dset::GssCredential;
using dset::GssCredentialManager;
using dset::CredentialSelectionCatalog;
using dset::Gsoap;
using dial::GsiAuthorizer;
using dial::FileGsiAuthorizer;
using dial::DialWs;
using dial::GsoapRegistry;

typedef std::vector<string> Libs;
typedef DialWs::TextList TextList;

//**********************************************************************
// Local definitions.
//**********************************************************************

void sigpipe_handler(int) {
  cout << "Another broken pipe" << endl;
}

//**********************************************************************

// Convert int to string.

string sint(int ival) {
  ostringstream ssval;
  ssval << ival;
  return ssval.str();
}
  
//**********************************************************************

// Return the working directory.

string workdir() {
  static string wdir = getcwd();
  return wdir;
}

//**********************************************************************

// Service start time.

Time service_start_time = Time::now();

//**********************************************************************

// Number of threads running services.

int& service_thread_count() {
  static int val = 0;
  return val;
}

//**********************************************************************

// Sum of service return statuses.

int& service_return_status() {
  static int val = 0;
  return val;
}

//**********************************************************************

// WS port.

int& ws_port() {
  static int val = 0;
  return val;
}

//**********************************************************************

// HTTP port.

int& http_port() {
  static int val = 0;
  return val;
}

//**********************************************************************

// WS authentication.

string& ws_auth() {
  static string val = "none";
  return val;
}

//**********************************************************************

// Service type.

string& service_type() {
  static string val;
  return val;
}

//**********************************************************************

// PID.

int& pid() {
  static int val = 0;
  return val;
}

//**********************************************************************

// Calls.

int& calls() {
  static int val = 0;
  return val;
}

int& serve_calls() {
  static int val = 0;
  return val;
}

int& deny_calls() {
  static int val = 0;
  return val;
}

int& error_calls() {
  static int val = 0;
  return val;
}

int& http_calls() {
  static int val = 0;
  return val;
}

//**********************************************************************

// Return the GSI authorizer.

#ifdef WITH_GSI
GsiAuthorizer& gsiauth() {
  string adir = workdir() + "/authorized_dn";
  static FileGsiAuthorizer auth(adir);
  return auth;
}

//**********************************************************************

// GSI authorization.

int service_authorization_callback(struct soap* psoap, char* dn) {
  //DialWs::lout() << "Begin GSI authorization..." << endl;
  struct gsi_plugin_data* pgsiplug =
    (struct gsi_plugin_data*) soap_lookup_plugin(psoap, GSI_PLUGIN_ID);
  // Fetch the identity string.
  assert( dn == pgsiplug->client_identity );
  string sid(dn);
  // Regularly update the list of authorized users.
  static Time update_time;
  Time new_time = Time::now();
  if ( TimeInterval(update_time, new_time).seconds() > 3600 ) {
    DialWs::lout() << "Updating authorization list" << endl;
    gsiauth().update();
    update_time = new_time;
  }
  string del = "unavailable";
  if ( pgsiplug->delegated_credential ) del = "available";
  // Check authorization.
  if ( gsiauth().is_authorized(sid) ) {
    DialWs::lout() << "GSI access granted: " << sid << endl;
    DialWs::lout() << "GSI delegated credential is " << del << endl;
    return 0;
  }
  DialWs::lout() << "GSI access denied: " << sid << endl;
  return 1;
}
#endif

//**********************************************************************

// Handle HTTP GET.

int http_get_handler(soap* psoap) {
  DialWs::lout() << ">>>HTTP GET" << endl;
  ++http_calls();
  string path = psoap->path;
  // Strip leading slashes.
  while ( path.size() && path[0]=='/' ) path = path.substr(1);
  // Strip trailing slashes.
  bool isdir = false;
  while ( path.size() && path[path.size()-1]=='/' ) {
    isdir = true;
    path = path.substr(0,path.size()-1);
  }
  // Display resolved path.
  DialWs::lout() << "Path: " << path << endl;
  // Retrieve page.
  Text tpg = DialWs::web_page(path);
  // Create response.
  soap_response(psoap, SOAP_HTML); // HTTP response header with text/html
  for ( int iline=0; iline<tpg.size(); ++iline ) {
    string sendline = tpg.line(iline) + "\n";
    soap_send(psoap, sendline.c_str());
  }
  soap_end_send(psoap);
  return SOAP_OK;
}

//**********************************************************************

// Condition to suspend service loops.
//   0 - run
//   1 - suspend
PThreadCondition& suspend_services() {
  static PThreadCondition val;
  return val;
}

//**********************************************************************

// Condition to inidcate initialization of services.
// Values:
//   0 - during intiialization
//   1 - initialization succeeded
//   2 - initialization failedded

PThreadCondition& initialization_condition() {
  static PThreadCondition con;
  return con;
}

//**********************************************************************

// Function to count active loops.
int& active_loop_count() {
  static int val = 0;
  return val;
}

//**********************************************************************

// Struct to communicate with threads.

class ThreadData {
public:
  int port;
  string auth;
  string svcname;
  int rstat;
};

//**********************************************************************

// Function to wait for threads.
// If called with an non-null argument that thread is added to the
// the list to join. If the argument is null, the function waits until
// all thread in its list have ended.
void join_threads(pthread_t thrid =0) {
  static vector<pthread_t> thrids;
  if ( thrid != 0 ) {
    thrids.push_back(thrid);
    return;
  }
  for ( vector<pthread_t>::const_iterator ithr=thrids.begin();
        ithr!=thrids.end(); ++ithr ) {
    cout << "Waiting for thread " << std::hex << *ithr << ": ";
    cout.flush();
    void* pval;
    int jstat = pthread_join(*ithr, &pval);
    cout << "  " << strerror(jstat) << endl;
  }
}

//**********************************************************************

// Routine to exit the following loop.

int exit_loop_with_error(int err) {
  if ( initialization_condition().is_locked() ) {
    assert( initialization_condition().lock_owner() != pthread_self() );
  }
  initialization_condition().lock();
  assert( initialization_condition().get() == 1 );
  initialization_condition().set(2);
  initialization_condition().unlock();
  initialization_condition().broadcast();
  return err;
}

//**********************************************************************

// Loop to handle requests on a port.
//
// port = port number
// auth = gsi, ssl or none
// svcname = name of the service to run

int dialws_loop(int port, string auth, string svcname) {

  // Fetch and check gsoap service.
  GsoapRegistry::Server* pserve = GsoapRegistry::server(svcname);
  struct Namespace* pnsp = GsoapRegistry::nspace(svcname);
  if ( pserve == 0 || pnsp == 0 )  {
    cerr << "Unable to set up service: " << svcname << endl;
    GsoapRegistry::display(cerr) << endl;
    return exit_loop_with_error(6);
  }

  // Initialize SOAP.
  int master = -1;   // master socket
  struct soap soap;
  soap_init(&soap);
  soap_set_namespaces(&soap, pnsp);

  // Enable port reuse.
  // Allows tests to succeed when they kill and restart server.
  // But there may be security implications (gsoap doc page 159).
  soap.bind_flags = SO_REUSEADDR;

  // Enable GSI.
#ifdef WITH_GSI
  // Register GSI plugin.
  struct gsi_plugin_data* pgsiplug =0;
  if ( auth == "gsi" ) {
    if ( soap_register_plugin(&soap, globus_gsi) ) {
      cerr << "Unable to register GSI plugin" << endl;
      soap_destroy(&soap);
      soap_end(&soap);
      soap_done(&soap);
      free(&soap);
      return exit_loop_with_error(7);
    }
    // Acquire credential.
    if ( gsi_acquire_credential(&soap) < 0 ) {
      cerr << "Unable to acquire credential" << endl;
      soap_destroy(&soap);
      soap_end(&soap);
      soap_done(&soap);
      free(&soap);
      return exit_loop_with_error(8);
    }
    //  Define authorization callback.
    pgsiplug =
      (struct gsi_plugin_data*) soap_lookup_plugin(&soap, GSI_PLUGIN_ID);
    pgsiplug->gsi_authorization_callback = service_authorization_callback;
  }
#endif

#ifdef WITH_OPENSSL
  // Setup SSL.
  int sslauthlevel = 0;
  //int sslauthlevel = SOAP_SSL_DEFAULT;
  char* serverpem = 0;
  char* password = 0;
  char* cacert = 0;
  char* capath = 0;
  char* dhfile = 0;
  char* randfile = 0;
  char* serverid = 0;
  int sstat = soap_ssl_server_context(
    &soap, sslauthlevel, serverpem, password,
    cacert, capath, dhfile, randfile, serverid);
  if ( sstat ) {
    cerr << "SSL setup failed with error " << sstat << endl;
    return exit_loop_with_error(9);
  }
#endif
  assert( port > 0 );
  if ( auth == "gsi" ) {
#ifdef WITH_GSI
    master = gsi_listen(&soap, 0, port, 100);
#else
    assert( auth != "gsi" );
#endif
  } else {
    master = soap_bind(&soap, NULL, port, 100);
  }
  if ( master < 0 ) {
    cerr << "SOAP bind returned " << master << endl;
    soap_print_fault(&soap, stderr);
    soap_destroy(&soap);
    soap_end(&soap);
    soap_done(&soap);
    free(&soap);
    return exit_loop_with_error(10);
  }
  DialWs::lout() << "Socket connection successful: socket "
                 << master << " listening on port " << port << endl;

  // Initialization.
  if ( svcname.size() ) {
    int istat = GsoapRegistry::init(svcname)();
    if ( istat != 0 ) {
      DialWs::lout() << "Service initialization failed with error "
                     << istat << endl;
      DialWs::lout() << "Service name is " << svcname << endl;
      return exit_loop_with_error(11);
    }
  }

  // Set the maximum time to wait for a client to respond.
  int msec = -1000;
  int sec = 1;
  soap.accept_timeout = 10*msec;
  soap.recv_timeout = 5*sec;
  soap.send_timeout = 5*sec;

  // Set function to handle web pages.
  soap.fget = http_get_handler; 

  // Loop over requests.
  int max_deny = 100;
  int deny_count = 0;
  ++active_loop_count();
  initialization_condition().lock();
  assert( initialization_condition().get() == 1 );
  initialization_condition().set(0);
  initialization_condition().unlock();
  initialization_condition().broadcast();
  while ( ! DialWs::terminate() ) {
    // Wait if we are paused.
    suspend_services().lock();
    while ( suspend_services().get() == 1 ) {
      suspend_services().wait(5);
    }
    suspend_services().unlock();
    // Begin loop iteration to handle a single request.
    struct soap* psoap = 0;
    bool authenticated = true;
    bool authorized = true;
    // Exit if there have been too many denials.
    if ( deny_count > max_deny ) {
      DialWs::lout() << "Too many denials in loop: " << deny_count << endl;
      break;
    }
    Time starttime;
    int connected_fd = -1;
#ifdef WITH_GSI
    string oldcredname = GssCredentialManager::name();
    // This will point to the archived copy of the delegated credential.
    GssCredential* pnewcred = 0;
    // This will carry the name associated with that credential.
    string newcredname;
#endif
    // Accept message from caller.
    // Loop until a request is found.
    bool terminate_service = false;
    while ( true ) {
      // Wait for loop condition to be available.
      DialWs::condition()->lock();
      while ( DialWs::condition()->get() ) {
        DialWs::condition()->wait(10);
      }
      DialWs::condition()->set(pthread_self());
#ifdef WITH_GSI
      if ( auth == "gsi" ) {
        connected_fd = gsi_accept(&soap);
      } else {
#else
      {
#endif
        connected_fd = soap_accept(&soap);
      }
      // If a request is found, act on it.
      if ( FileStatus("debug_dialws").exists() ) {
        cout << "dialws: " << *DialWs::condition() << endl;
      }
      if ( connected_fd != -1 ) break;
      // Unset condition.
      DialWs::condition()->set(0);
      DialWs::condition()->unlock();
      DialWs::condition()->broadcast();
      sched_yield();
      // Exit loop for termination request.
      if ( DialWs::terminate() ) {
        DialWs::lout() << "Terminating service on port " << master << endl;
        terminate_service = true;
        break;
      }
    }
    if ( terminate_service ) {
      break;
    }
    ++calls();
    starttime = Time::now();
    struct tm tmptm;
    DialWs::lout() << endl;
    DialWs::lout() << "****************** Begin " << auth << " request  "
                   << starttime << " ******************" << endl;
    if ( connected_fd < 0 ) {
      DialWs::lout() << "Error occurred while accepting connection"
                     << endl;
      authorized = false;
      ++error_calls();
      continue;
    } else {
      psoap = soap_copy(&soap);
      if ( psoap == 0 ) {
        DialWs::lout() << "dialws: soap_copy failed: "
                       << "Too many connections???" << endl;
        ++error_calls();
        continue;
      }
    }
    // Check for SSL.
    if ( auth == "ssl" ) {
#ifdef WITH_OPENSSL
      if ( soap_ssl_accept(psoap) ) {
        DialWs::lout() << "Unable accept SSL" << endl;
        soap_print_fault(psoap, stderr);
        authorized = false;
        ++error_calls();
        continue;
      } else {
        DialWs::lout() << "Accepted SSL" << endl;
      }
#else
      cerr << "SSL is not enabled" << endl;
      assert(false);
#endif
    }
#ifdef WITH_GSI
    if ( auth == "gsi" ) {
      pgsiplug = (struct gsi_plugin_data*)
        soap_lookup_plugin(psoap, GSI_PLUGIN_ID);
      if ( gsi_accept_security_context(psoap) == 0 ) {
        authorized = pgsiplug->gsi_authorization_callback(psoap, pgsiplug->client_identity) == 0;
        // If there is a delegated credential, then record it and
        // set it as the default for this thread.
        bool noisy = true;
        Gsoap wsoap(*psoap);
        GssCredential delcred = wsoap.delegated_credential();
        if ( delcred.is_valid() ) {
          // Record credential.
          // This will replace any earlier credential with the
          // same name.
          int cstat = GssCredentialManager::insert(delcred);
          if ( cstat ) {
            DialWs::lout() << "Unable to record delegated credential"        
                           << endl;
          } else {
            pnewcred = GssCredentialManager::get(delcred.name());
            assert( pnewcred != 0 );
            if ( ! pnewcred->is_valid() ) {
              DialWs::lout() << "Delegated credential is not valid"
                             << endl;
            } else {
              // Set this as the credential to use for this thread.
              assert( pnewcred->name() == delcred.name() );
              newcredname = pnewcred->name();
              cstat = GssCredentialManager::set_name(newcredname);
              if ( cstat ) {
                newcredname = "";
                DialWs::lout() << "Unable to set thread credential" << endl;
              } else {
                assert( GssCredentialManager::credential() == pnewcred );
              }
            }
          }
        } else {
          GssCredentialManager::set_name(pgsiplug->client_identity);
        }
        if ( noisy ) {
          if ( newcredname.size() ) {
            DialWs::lout() << "Credential has been delegated" << endl;
          } else {
            DialWs::lout() << "Unable to delegate credential" << endl;
          }
        }
      } else {
        DialWs::lout() << "Unable to estabish security context." << endl;
        authenticated = false;
        authorized = false;
      }
    }
#endif
    if ( authorized ) {
      ThreadCredential tc;
      DialWs::lout() << "Credential name: " << tc.name() << endl;
      DialWs::lout() << "Credential owner: " << tc.owner() << endl;
      // Ignore request and set terminate flag if the file
      // terminate_service is found.
      if ( DialWs::terminate() ) {
        DialWs::lout() << "Ignoring request and terminating service"
                       << endl;
      // Otherwise, serve request.
      } else {
        deny_count = 0;
        ++serve_calls();
        pserve(psoap);
      }
    // Otherwise deny access.
    } else {
      if ( ! authenticated ) {
        DialWs::lout() << "Denying unauthenticated request" << endl;
      } else {
        DialWs::lout() << "Denying unauthorized request" << endl;
      }
      ++deny_count;
      ++deny_calls();
      DialWs::lout() << "Deny count is " << deny_count << "/" << max_deny
                     << endl;
      soap_receiver_fault(psoap, "Access denied", "");
      soap_send_fault(psoap);
    }
    if ( psoap != 0 ) {
      soap_destroy(psoap);
      soap_end(psoap);
      soap_done(psoap);
      free(psoap);
    }
#ifdef WITH_GSI
    // Restore the original credential name for this thread.
    if ( newcredname.size() ) {
      int cstat = GssCredentialManager::set_name(oldcredname);
      if ( cstat ) {
        newcredname = "";
        DialWs::lout() << "Unable to restore thread credential" << endl;
      }
    }
#endif
    // Display event trailer.
    Time endtime = Time::now();
    TimeInterval dtime(starttime, endtime);
    DialWs::lout() << "************** End request  " << endtime
                   << " (" << dtime << " elapsed) ************" << endl;
    // Unset condition.
    DialWs::condition()->set(0);
    DialWs::condition()->unlock();
    DialWs::condition()->broadcast();
  }
  --active_loop_count();

  DialWs::lout() << endl;
  DialWs::lout() << "Server terminating (socket " << master << ")" << endl;

  return 0;
} 

//**********************************************************************

// Function to create thread to loop over requests on a port.

PThreadMutex& start_mutex() {
  static PThreadMutex val;
  return val;
}

void* service_thread(void* pvdata) {
  //pthread_detach(pthread_self());
  // Copy the thread data.
  ThreadData* pdata = (ThreadData*) pvdata;
  int port = pdata->port;
  string auth = pdata->auth;
  string svcname = pdata->svcname;
  pdata->rstat = 0;
  cout << svcname << " service thread: " << pthread_self() << endl;
  // Start loop.
  int rstat = dialws_loop(port, auth, svcname);
  --service_thread_count();
  service_return_status() += rstat;
  return pdata;
}

//**********************************************************************

// Function to start a thread to loop over requests on a port.

int start_service_thread(int port, string auth, string svcname) {
  cout << "  port: " << port << endl;
  cout << "  auth: " << auth << endl;
  cout << "  name: " << svcname << endl;
  // Fill thread data.
  pthread_t thrid;
  ThreadData data;
  data.port = port;
  data.auth = auth;
  data.svcname = svcname;
  data.rstat = 1;
  // Increment service count.
  ++service_thread_count();
  // Start thread.
  initialization_condition().lock();
  initialization_condition().set(1);
  int rstat = pthread_create(&thrid, 0, service_thread, &data);
  join_threads(thrid);
  if ( rstat == 0 ) {
    // Wait for thread to signal initialization is complete.
    Time start = Time::now();
    while ( initialization_condition().get() ) {
      TimeInterval wait(start, Time::now());
      cout << "dialws: Waiting for " << svcname
           << " service to initialize ("  << wait << ")" << endl;
      initialization_condition().wait(60);
      if ( initialization_condition().get() == 2 ) {
        rstat = 2;
        break;
      }
    }
    if ( rstat == 0 ) {
      cout << "dialws: Initialization complete" << endl;
    } else {
      cout << "dialws: Initialization failed" << endl;
    }
  } else {
    cout << "dialws: ERROR: Unable to create service thread" << endl;
  }
  initialization_condition().unlock();
  return rstat;
}

//**********************************************************************

// Function to create home page.

void create_home_page(string path, string) {
  Text wpg;
  wpg.append("<HTML>");
  wpg.append("<H2>DIAL web service</H2>");
  wpg.append("Type: " + service_type());
  // HTTP URL.
  ostringstream ss_http_url;
  ss_http_url << "http://" << get_hostname() << ":" << http_port();
  wpg.append("<br>HTTP URL: " + ss_http_url.str());
  // WS URL.
  if ( ws_port() > 0 ) {
    ostringstream ss_ws_url;
    ss_ws_url << "http://" << get_hostname() << ":" << ws_port();
    wpg.append("<br>WS URL: " + ss_ws_url.str());
    wpg.append("<br>WS authentication: " + ws_auth() );
  } else {
    wpg.append("<br>Web service is disabled");
  }
  // Username.
  /*
  char chusername[128];
  getlogin_r(chusername, 128);
  string username = chusername;
  wpg.append("<br>Owner: " + username);
  */
  // PID.
  ostringstream sspid;
  sspid << pid();
  wpg.append("<table border=\"0\" cellspacing=\"0\">");
  wpg.append("<tr><td align=\"right\">Host: <td>" + get_hostname());
  wpg.append("<tr><td align=\"right\">PID: <td>" + sspid.str());
  wpg.append("</table>");
  // Start time.
  wpg.append("<table border=\"0\" cellspacing=\"0\">");
  wpg.append("<tr><td align=\"right\">Start time: <td>" +
             service_start_time.to_string() + " UTC");
  // Current time.
  Time cur_time = Time::now();
  wpg.append("<tr><td align=\"right\">Current time: <td>" +
             cur_time.to_string() + " UTC");
  wpg.append("</table>");
  // Up time.
  wpg.append("Up time: " +
             TimeInterval(service_start_time, cur_time).to_string());
  // Trailer.
  wpg.append("<pre>");
  wpg.append("# calls: " + sint(calls()));
  wpg.append(" served: " + sint(serve_calls()));
  wpg.append(" denied: " + sint(deny_calls()));
  wpg.append(" errors: " + sint(error_calls()));
  wpg.append("   http: " + sint(http_calls()));
  wpg.append("</pre>");
  // Home page insertions.
  TextList hpis = DialWs::home_page_insertions();
  for ( TextList::const_iterator ihpi=hpis.begin();
        ihpi!=hpis.end(); ++ihpi ) {
    const Text& hpi = *ihpi;
    wpg.append("");
    wpg.append("<!-- home page insertion -->");
    for ( int iline=0; iline<hpi.size(); ++iline ) {
      wpg.append(hpi.line(iline));
    }
  }
  wpg.append("");
  wpg.append("</HTML>");
  DialWs::insert_web_page("", "", wpg);
}

//**********************************************************************
// Main.
//**********************************************************************

int main(int argc, char **argv) {
 
  signal(SIGPIPE, sigpipe_handler);

  cout << "Main thread: " << pthread_self() << endl;

  // Read command line.
  string exename = argv[0];
  cout << exename << endl;
  for ( int iarg=1; iarg<argc; ++iarg ) {
    cout << "  " << argv[iarg] << endl;
  }

  // Read svclib:svcname and decompose.
  Libs libs;
  string svcpath = argv[1];
  string svclist = svcpath;
  while ( true ) {
    string::size_type isep = svclist.find(':');
    if ( isep == string::npos ) break;
    string svclib = svclist.substr(0, isep);
    libs.push_back(svclib);
    svclist = svclist.substr(isep+1);
  }
  service_type() = svclist;

  // Read the port.
  http_port() = 8080;
  string logfile = "";
  if ( argc > 2 ) {
    string arg1 = argv[2];
    if ( arg1 == "-h" ) {
      cout << "Usage: " << exename 
           << " [LIBNAME:]SVCNAME [PORT] [LOGFILE] [gsi|none]" << endl;
      return 0;
    } else {
      istringstream sport(arg1);
      sport >> http_port();
      if ( http_port <= 0 ) {
        cerr << "Invalid port number " << http_port() << endl;
        return 1;
      }
      // Fetch the log file. If undefined, there is no logging.
      if ( argc > 3 ) {
        logfile = argv[3];
      }
    }
  }

  // Read the authorization.
  ws_auth() = "gsi";
  if ( argc > 4 ) {
    string auth = argv[4];
    if ( auth != "none" && auth != "gsi" && auth != "ssl" ) {
      cerr << "Invalid authorization option: " << auth << endl;
      cerr << "  Use NONE, GSI or SSL" << endl;
      return 2;
    }
    ws_auth() = auth;
  }
#ifndef WITH_GSI
  if ( ws_auth() == "gsi" ) {
    cerr << "GSI is not supported" << endl;
    return 3;
  }
#endif

  // Read the thread option.
  string thropt = "service";
  //if ( ws_auth() == "gsi" ) {
  //  thropt = "http_service";
  //}
  if ( argc > 5 ) {
    thropt = argv[5];
  } 

  assert( argc <= 6 );

  // Assign WS port.
  if ( thropt == "http_service" ) {
    ws_port() = http_port() + 1;
  } else if ( thropt == "http" ) {
    ws_port() == 0;
  } else {
    ws_port() = http_port();
  }
  if ( http_port() == 0 ) {
    cout << "dialws: No HTTP port." << endl;
  } else {
    cout << "dialws: HTTP port: " << http_port() << endl;
  }
  if ( ws_port() == 0 ) {
    cout << "dialws: No WS port." << endl;
  } else {
    cout << "dialws: WS port: " << ws_port() << endl;
  }

  // Check that a server is not already running in this directory.
  if ( FileStatus("pid").exists() ) {
    int oldpid;
    ifstream pidfile("pid");
    pidfile >> oldpid;
    cerr << "Process " << oldpid << " did not terminate properly" << endl;
    cerr << "Kill it and delete pid" << endl;
    return 4;
  }

  // Log PWD.
  cout << "dialws: PWD: " << getcwd() << endl;

  // Write the host.
  string host = get_hostname();
  cout << "dialws: Host: " << host << endl;

  // Remove service termination flag.
  unlink("terminate_service");

  // Open logfile.
  cout << "dialws: Opening logfile " << logfile << endl;
  DialWs::lout(logfile) << "Opening server log " << logfile << endl;
  cout << "dialws: ";
  if ( ws_auth() == "gsi" ) {
    cout << "GSI ";
  } else {
    cout << "Unauthenticated ";
  }
  cout << "WS server on port " << ws_port() << endl;
  cout << "dialws: ";
  cout << "HTTP server on port " << http_port() << endl;
  cout << "dialws: ";
  cout << "Exe: " << exename << endl;
  cout << "dialws: ";
  cout << "Dir: " << workdir() << endl;

  // Open libraries.
  cout << "dialws: Opening libraries" << endl;
  for ( Libs::const_iterator
        ilib=libs.begin(); ilib!=libs.end(); ++ilib ) {
    string svclib = *ilib;
    cout << "Opening library " << svclib << endl;
    string lib = svclib;
    if ( lib.find(".so") == string::npos ) {
      lib = "lib" + lib + ".so";
    }
    void* handle = dlopen(lib.c_str(), RTLD_LAZY|RTLD_GLOBAL);
    if ( handle == 0 ) {
      cout << "Unable to open library " << lib << endl;
      cout << dlerror() << endl;
      return 5;
    }
  }

  // Register function to create web page.
  DialWs::insert_web_page_update("", create_home_page);

  // Create a condition that the service threads can use for mutual
  // exclusion. Not clear if this isreally needed.
  // Service types (e.g. scheduler) may choose to override this.
  PThreadCondition* psvccond = new PThreadCondition;
  DialWs::condition() = psvccond;

  // Load credential name-owner map and check CSC.
  cout << "Credential selection catalog:" << endl;
  cout << CredentialSelectionCatalog::default_instance() << endl;
  if ( ! CredentialSelectionCatalog::default_instance().is_valid() ) {
    cout << "Credential selection catalog is invalid" << endl;
    cout << "  Credential owners may not be assigned." << endl;
    cout << "  This may interfere with authorization." << endl;
  }

  // Start service loop.
  cout << "dialws: Create service threads with option " << thropt << endl;
  int rstat = 0;
  int tstat = 0;
  // Suspend service loops before startup.
  suspend_services().lock();
  suspend_services().set(1);
  suspend_services().unlock();
  // Single port, no thread.
  if ( thropt == "nothread" ) {
    cout << "dialws: Starting no-thread service" << endl;
    service_return_status() +=
      dialws_loop(ws_port(), ws_auth(), service_type());
  // Single port with thread.
  } else if ( thropt == "service" ) {
    cout << "dialws: Starting service thread" << endl;
    assert( http_port() == ws_port() );
    tstat += start_service_thread(ws_port(), ws_auth(), service_type());
  // HTTP only, with thread.
  } else if ( thropt == "http" ) {
    cout << "dialws: Starting HTTP thread" << endl;
    string svctype = service_type() + "_http";
    if ( ws_auth() == "gsi" ) {
      cout << "dialws:  Ignoring request to use GSI" << endl;
    }
    tstat += start_service_thread(http_port(), "none", svctype);
  // HTTP and WS in separate threads.
  } else if ( thropt == "http_service" ) {
    cout << "dialws: Starting separate HTTP and service threads"
           << endl;
    string wwwtype = service_type() + "_http";
    cout << "dialws: ...HTTP thread";
    tstat += start_service_thread(http_port(), "none", wwwtype);
    cout << "dialws: ...service thread";
    tstat += start_service_thread(ws_port(), ws_auth(), service_type());
  // Error.
  } else {
    cout << "dialws: Unknown thread option: " << thropt << endl;
    rstat = 99;
  }

  // Check that services started OK.
  if ( rstat != 0 ) {
    cout << "dialws: Error starting service: " << rstat << endl;
    return rstat;
  }
  // Check that services started OK.
  if ( tstat != 0 ) {
    cout << "dialws: Error starting service thread: " << tstat << endl;
    return tstat;
  }

  // Wait for all threads to become active.
  for ( int count=0; true; ++count ) {
    cout << "dialws: "
         << active_loop_count() << " of " << service_thread_count()
         << " services are active." << endl;
    if ( active_loop_count() == service_thread_count() ) {
      break;
    }
    pthread_sleep(10);
    if ( count > 100 ) {
      cout << "dialws: Giving up" << endl;
      return(88);
    }
  }

  // Start processing.
  suspend_services().set(0);
  suspend_services().broadcast();

  // Write the PID.
  pid() = getpid();
  cout << "PID: " << pid() << endl;
  ofstream pidfile("pid");
  pidfile << pid() << endl;
  pidfile.flush();   // Hold on to the file.

  // Wait for service termination and handle pause/resume.
  bool dbg = true;
  suspend_services().lock();
  while ( service_thread_count() > 0 ) {
    if ( dbg ) {
      cout << "dialws: "
           << active_loop_count() << " of " << service_thread_count()
           << " services are active." << endl;
    }
    dbg = FileStatus("DEBUG_dialws").exists();
    if ( FileStatus("terminate").exists() ) {
      suspend_services().set(0);
      break;
    }
    if ( suspend_services().get() &&
         ! FileStatus("suspend").exists() ) {
      cout << "dialws: Resuming services" << endl;
      suspend_services().set(0);
      suspend_services().broadcast();
    }
    if ( ! suspend_services().get() &&
         FileStatus("suspend").exists() ) {
      cout << "dialws: Pausing services" << endl;
      suspend_services().set(1);
    }
    // Wait for a signal that should never come.
    suspend_services().wait(10);
  }
  suspend_services().unlock();

  // Wait for all threads to become inactive.
  for ( int count=0; true; ++count ) {
    cout << "dialws: "
         << active_loop_count() << " of " << service_thread_count()
         << " services are active." << endl;
    if ( active_loop_count() == 0 ) {
      break;
    }
    pthread_sleep(10);
    if ( count > 10 ) {
      cout << "dialws: Giving up" << endl;
      return(89);
    }
  }

  // Check the service condition.
  assert( psvccond != 0 );
  assert( ! psvccond->is_locked() );
  delete psvccond;

  // Wait for threads to end.
  join_threads();

  int sstat = service_return_status();
  cout << "dialws: Exited service loop with status " << sstat << endl;
  unlink("pid");
  return sstat;
}

//**********************************************************************