ParaViewWeb, Python Launcher, and Apache, and 'Process did not properly start' Error

Hello,

I’m working on spinning up an instance of ParaViewWeb Python Launcher with Apache as a front-end as described in your documentation, and am running into some confounding issues.

Basically, my current local setup is as follows:

Apache is set up to surface my server as a virtual host

<VirtualHost *:80>
    ServerName localhost.paraview
    #ServerAdmin webmaster@example-host.example.com
    DocumentRoot "{project_directory}/www/"
    ErrorLog "{logging_directory}/apache2/localhost.paraview-error_log"
    CustomLog "{logging_directory}/apache2/localhost.paraview-access_log" common
    Header set Access-Control-Allow-Origin "*"

    ### The following commented lines could be useful when running
    ### over https and wss:
    # SSLEngine On
    # SSLCertificateFile    /etc/apache2/ssl/your_certificate.crt
    # SSLCertificateKeyFile /etc/apache2/ssl/your_domain_key.key
    # SSLCertificateChainFile /etc/apache2/ssl/DigiCertCA.crt
    #
    # <Location ${MY-DOCUMENT-ROOT} >
    #   SSLRequireSSL On
    #   SSLVerifyClient optional
    #   SSLVerifyDepth 1
    #   SSLOptions +StdEnvVars +StrictRequire
    # </Location>

    # Rule for ParaViewWeb launcher
    # ProxyPass /paraview http://localhost.paraview
    ProxyPass /paraview http://localhost:8080/paraview

    # Rewrite setup for ParaViewWeb
    RewriteEngine On

    # This is the path the mapping file Jetty creates
    RewriteMap session-to-port txt:{project_directory}/mapping-dir/proxy-mapping.txt

    # This is the rewrite condition. Look for anything with a sessionId=
    # in the query part of the URL and capture the value to use below.
    RewriteCond %{QUERY_STRING}     ^sessionId=(.*)&path=(.*)$ [NC]

    # This does the rewrite using the mapping file and the sessionId
    RewriteRule    ^/proxy.*$  ws://${session-to-port:%1}/%2  [P]

    <Directory "{project_directory}/www/">
        Options Indexes FollowSymLinks
        Order allow,deny
        Allow from all
        AllowOverride None
        #AllowOverride All
        Require all granted
    </Directory>
</VirtualHost>

where {logging_directory} and {project_directory} are paths on my system and everything else above is given directly as in my vhosts file.

Additionally my launcher.config is

{
  // ===============================
  // General launcher configuration
  // ===============================

  "configuration": {
    "host" : "localhost",
    "port" : 8080,
    "endpoint": "paraview",                   // SessionManager Endpoint
    //"content": "./www",                    // Optional: Directory shared over HTTP
    "proxy_file" : "./mapping-dir/proxy-mapping.txt",  // Proxy-Mapping file for Apache
    "sessionURL" : "ws://localhost:8080/proxy?sessionId=${id}&path=ws",
         // ws url used by the client to connect to the started process
    "timeout" : 25,                           // Wait time in second after process start
    "log_dir" : "./viz-logs",              // Directory for log files
    //"upload_dir" : "/.../data",               // If launcher should act as upload server, where to put files
    "fields" : ["file", "host", "port", "updir"] //,    // List of fields that should be send back to client
                                                     // include "secret" if you provide it as an --authKey to the app
    //"sanitize": {                             // Check information coming from the client
    //  "cmd": {
    //    "type": "inList",                 // 'cmd' must be one of the strings in 'list'
    //    "list": [
    //      "me", "you", "something/else/altogether", "nothing-to-do"
    //    ],
    //    "default": "nothing-to-do"        // If the string doesn't match, replace it with the default.
                                        // Include the default in your list
      //},
      //"cmd2": {                             // 'cmd2' must match the regexp provided, example: not a quote
      //  "type": "regexp",
      //  "regexp": "^[^"]*$",             // Make sure to include '^' and '$' to match the entire string!
      //  "default": "nothing"
      //}
    //}
  },

  // ===============================
  // Useful session vars for client
  // ===============================

  "sessionData" : { "updir": "/Home" },      // Tells client which path to updateFileBrowser after uploads

  // ===============================
  // Resources list for applications
  // ===============================

  "resources" : [ { "host" : "localhost", "port_range" : [9001, 9003] } ],

  // ===============================
  // Set of properties for cmd line
  // ===============================

  "properties" : {
    "vtkpython" : "/.../VTK/build/bin/vtkpython",
    "pvpython" : "/.../ParaView/build/bin/pvpython",
    "vtk_python_path": "/.../VTK/build/Wrapping/Python/vtk/web",
    "pv_python_path": "/.../ParaView/build/lib/site-packages/paraview/web",
    "plugins_path": "/.../ParaView/build/lib",
    "dataDir": "/.../path/to/data/directory"
  },

  // ===============================
  // Application list with cmd lines
  // ===============================

  "apps" : {
    "cone" : {
      "cmd" : [
        "${vtkpython}", "${vtk_python_path}/vtk_web_cone.py", "--port", "$port"
      ],
      "ready_line" : "Starting factory cone"
    },
    "graph" : {
      "cmd" : [
          "${vtkpython}", "${vtk_python_path}/vtk_web_graph.py", "--port", "$port",
            "--vertices", "${numberOfVertices}", "--edges", "${numberOfEdges}"
      ],
      "ready_line" : "Starting factory graph"
    },
    "phylotree" : {
      "cmd" : [
        "${vtkpython}", "${vtk_python_path}/vtk_web_phylogenetic_tree.py", "--port", "$port",
          "--tree", "${dataDir}/visomics/${treeFile}", "--table", "${dataDir}/visomics/${tableFile}"
      ],
      "ready_line" : "Starting factory phylotree"
    },
    "filebrowser" : {
      "cmd" : [
        "${vtkpython}", "${vtk_python_path}/vtk_web_filebrowser.py",
          "--port", "${port}", "--data-dir", "${dataDir}"
      ],
      "ready_line" : "Starting factory filebrowser"
    },
    "data_prober": {
      "cmd": [
        "${pvpython}", "-dr", "${pv_python_path}/pv_web_data_prober.py",
          "--port", "${port}", "--data-dir", "${dataDir}", "-f"
      ],
      "ready_line" : "Starting factory data_prober"
    },
    "visualizer": {
      "cmd": [
        "${pvpython}", "-dr", "${pv_python_path}/pv_web_visualizer.py",
          "--plugins", "${plugins_path}/libPointSprite_Plugin.so", "--port", "${port}",
          "--data-dir", "${dataDir}", "--load-file", "${dataDir}/${fileToLoad}",
          "--authKey", "${secret}", "-f"
      ],  // Use of ${secret} means it needs to be provided to the client, in "fields", above.
      "ready_line" : "Starting factory visualizer"
    },
    "loader": {
      "cmd": [
        "${pvpython}", "-dr", "${pv_python_path}/pv_web_file_loader.py",
          "--port", "${port}", "--data-dir", "${dataDir}",
          "--load-file", "${dataDir}/${fileToLoad}", "-f"
      ],
      "ready_line" : "Starting factory loader"
    },
    "launcher" : {
      "cmd": [
        "/.../ParaView/Web/Applications/Parallel/server/launcher.sh",
          "${port}", "${client}", "${resources}", "${file}"
      ],
      "ready_line" : "Starting factory launcher"
    },
    "server": {
      "cmd": [
        //"your_shell_script.sh", "--resource-host", "${host}", "--resource-port", "${port}",
        //  "--session-id", "${id}", "--generated-password", "${secret}",
        //  "--application-key", "${application}"
        //"${python_exec}", "-dr", "./server/Server.py"
        "/Applications/ParaView-5.7.0-RC2.app/Contents/bin/pvpython {project_directory}/server/Server.py -p", "${port}"
      ],
      "ready_line": "Starting factory SERVER"
    }
  }

where again {project_directory} is a path on my system and everything else above is given directly as in my config file, and the ‘server’ command is the one that I’m trying to trigger from the launcher, to open multiple versions of the app for different users from a base login screen.

To use my ParaViewWeb app, I start up the launcher with a script in a terminal and then go to ‘localhost.paraview’, a base log in screen Apache serves up, and then ideally, at the push of a ‘Continue as guest’ button, the launcher should be triggered to start up a basic implemenation of my app at ‘localhost.paraview/paraview’, and the browser is forwarded there.

I can see by the mapping-dir/proxy-mapping.txt and the viz-logs/ that the launcher is attempting to trigger the start up of the app, and that the ports/ids are being apportioned, and the browser is forwarding to ‘localhost.paraview/paraview’ but I keep receiving the following error message:

In browser:

Request URL: http://localhost:8080/paraview
Request Method: POST
Status Code: 503 Service Unavailable

Request Payload : {
  sessionManagerURL: "http://localhost:8080/paraview", 
  application: "server", 
  app: "server"
}

Response: {
  "error": "The process did not properly start. ['/Applications/ParaView-5.7.0-RC2.app/Contents/bin/pvpython /Development/labkey_em_viewer/server/Server.py -p', '9002']"
}

And in the viz-logs:

2020-12-18 17:11:23,607:ERROR:root:The command line failed
2020-12-18 17:11:23,609:ERROR:root:/Applications/ParaView-5.7.0-RC2.app/Contents/bin/pvpython /Development/labkey_em_viewer/server/Server.py -p 9002

I’m not sure what’s going on here, if I’m not triggering the app from the launcher in the right way, or if apache isn’t interacting with the launcher properly, etc. I’ve trued several different strategies and I keep ending up back here.

Could anyone help me determine what sort of errors cause the launcher to behave this way?

Thanks,
GM

You should try to deploy that already done demo available here. You will notice how the launcher config is setup. Please fix your path in that configuration file to match the inside docker path.

You might be interested to read that thread too.

Hi there,

Thank you @jourdain for those pointers, it was really helpful to getting a working installation of the launcher running for reference.

I’ve gotten the demo of yours running on my system, and it’s working great. (Though I can only see the demo at ‘www.example.com:9000’ if i add 127.0.0.1 www.example.com to my /etc/hosts file, if that is an indication of some other misconfiguration, that would be good to know, otherwise it appears and operates normally at ‘localhost:9000’.)

I’ve used it as you suggested in this format and I can see in my log files that the launcher is starting and apportioning sessions and ports to use:

2021-02-25 00:32:05,379:INFO:twisted:Site starting on 9000
2021-02-25 00:32:05,380:INFO:twisted:Starting factory <twisted.web.server.Site object at 0x7f67ef7a4240>

In the browser a ‘Loading Paraview…’ message for about a minute, before the ‘Server Disconnected’ message again comes up, with either an

WebSocket connection to 'ws://example.com:9000/proxy?sessionId=7b4ac654-7701-11eb-8dff-0242ac110002&path=ws' failed: Error in connection establishment: net::ERR_NAME_NOT_RESOLVED

Or an undetermined ‘503’ error, depending on whether I have 127.0.0.1 example.com to my /etc/hosts file or not (no host file additions → ERR_NAME_NOT_RESOLVED, with host file additions → 503):

WebSocket connection to 'ws://example.com:9000/proxy?sessionId=2eed1fea-7702-11eb-b174-0242ac110002&path=ws' failed: Error during WebSocket handshake: Unexpected response code: 503

And the connection times out:

2021-02-25 00:42:22,022:INFO:twisted:"127.0.0.1" - - [25/Feb/2021:00:42:21 +0000] "POST /paraview HTTP/1.1" 200 211 "http://localhost:9000/" "..."
2021-02-25 00:43:22,045:INFO:twisted:Timing out client: IPv4Address(type='TCP', host='127.0.0.1', port=42990)

As a result, I have been digging deeply into exactly what my app is trying to do that is conflicting with the launcher, since it seems clear the launcher is working as expected. What I have realized is this, my app is designed based on the RemoteRenderer example which, I believe, is calling the wslink server directly rather than using the launcher. I am beginning to think I am running into a collision of wslink servers, or perhaps that the issue is that the launcher has no capacity to override the ‘protocol’ argument as exemplified for a RemoteRenderer.

Is there any documentation or examples on how to reconcile these different approaches? Do you have any suggestions?

Thanks, GM

Just change that part in the launcher config so the proper port get used when connecting back to the ws.

HTH

Hi @jourdain ,

I tried replacing the line in the launcher config today per your suggestion:

"sessionURL" : "SESSION_URL_ROOT/proxy?sessionId=${id}&path=ws",

to

"sessionURL" : "ws://example.com/proxy?sessionId=${id}&path=ws"

and it seems to have taken me backward rather than forward.

My launcher config in full today is:

{
  // ===============================
  // General launcher configuration
  // ===============================

  "configuration": {
    "log_dir" : "/pvw/launcher/log",
    "host" : "localhost",
    "endpoint": "paraview",
    "sessionURL" : "SESSION_URL_ROOT/proxy?sessionId=${id}&path=ws",
    "timeout" : 100,
    //"upload_dir" : "/.../data",
    "fields" : [],
    "port" : 9000,
    "proxy_file" : "/opt/launcher/proxy-mapping.txt",

    "sanitize": {
      "version": {
          "type": "regexp",
          "regexp": "^v[0-9]+.[0-9]+.[0-9]+$",
          "default": "v0.0.0"
      },
      "file": {
          "type": "regexp",
          "regexp": "^[-\\\\w./]+$",
          "default": "emptyFile"
      }
    }
  },

  // ===============================
  // Useful session vars for client
  // ===============================

  "sessionData" : { "updir": "/Home" },      // Tells client which path to updateFileBrowser after uploads

  // ===============================
  // Resources list for applications
  // ===============================

  "resources" : [ { "host" : "localhost", "port_range" : [9010, 9014] } ],

  // ===============================
  // Set of properties for cmd line
  // ===============================

  "properties": {
    "dataDir": "/data",
    "webapps_dir": "/opt/paraview/share/paraview-5.7/web",
    "python_exec": "/opt/paraview/bin/pvpython"
  },

  // ===============================
  // Application list with cmd lines
  // ===============================

  "apps" : {
    "cone" : {
      "cmd" : [
        "${vtkpython}", "${vtk_python_path}/vtk_web_cone.py", "--port", "$port" 
      ],
      "ready_line" : "Starting factory cone"
    },
    "visualizer": {
      "cmd": [
        "${python_exec}",
        "${webapps_dir}/visualizer/server/pvw-visualizer.py",
        "--port", "${port}",
        "--data", "${dataDir}",
        "--authKey", "${secret}",
        "--viewport-max-width", "1920",
        "--viewport-max-height", "1080",
        "--timeout", "30"
      ],
      "ready_line" : "Starting factory"
    },
    "server": {
      "cmd": [
        "${python_exec}", "/pvw/server/Server.py", "-p", "${port}"
      ],
      "ready_line": "Startingsss factory SERVER"
    }
  }
}

with the alterations to match the docker container setup. I am then calling it with:

  config = {
    'sessionManagerURL' : 'http://localhost:9000/paraview',
    'application' : 'server',
    'app' : 'server'
  }

Any further thoughts?

Thank you, GM

You should have kept SESSION_URL_ROOT but you should use the arg/env to change its content dynamically.

What I told you still stand. The returned sessionURL is wrong and prevent the client to connect to the correct machine (host/port).

It seems that you are reaching the service on port 9000 therefore you should have kept that port.

In other word if your URL looks like: http://HOST:PORT/ the sessionURL should resolve to ws://HOST:PORT/proxy?sessionId=${id}&path=ws.

HTH