## SLURM PROLOG ############################################################### ## Job ID : 15666868 ## Job Name : pv-tunnel ## Nodelist : gpu[1210-1211] ## CPUs : 8 ## Mem/CPU : 40960 MB ## Directory : /gpfs/data/ccvstaff/cdiaz8/remote_render_logs ## Started : Wed Nov 18 11:40:48 EST 2020 ############################################################################### -np 2 /gpfs/runtime/opt/paraview/5.8.0_release/bin/pvserver -display :0.0 --force-offscreen-rendering : -np 2 /gpfs/runtime/opt/paraview/5.8.0_release/bin/pvserver -display :0.1 --force-offscreen-rendering : -np 2 /gpfs/runtime/opt/paraview/5.8.0_release/bin/pvserver -display :0.2 --force-offscreen-rendering : -np 2 /gpfs/runtime/opt/paraview/5.8.0_release/bin/pvserver -display :0.3 --force-offscreen-rendering : module: unloading 'python/3.7.4' module: loading 'python/3.7.4' gpu1210.oscar.ccv.brown.edu module: loading 'mpi/openmpi_4.0.4_gcc' module: mpi: To use: module load gcc/8.3 Init PServer -------------------------------------------------------------------------- The following command line options and corresponding MCA parameter have been deprecated and replaced as follows: Command line options: Deprecated: --bynode, -bynode Replacement: --map-by node Equivalent MCA parameter: Deprecated: rmaps_base_bynode Replacement: rmaps_base_mapping_policy=node The deprecated forms *will* disappear in a future version of Open MPI. Please update to the new syntax. -------------------------------------------------------------------------- Waiting for client... Connection URL: cs://gpu1210.oscar.ccv.brown.edu:11111 Accepting connection(s): gpu1210.oscar.ccv.brown.edu:11111 [gpu1210.oscar.ccv.brown.edu:81563] 1 more process has sent help message help-orte-rmaps-base.txt / deprecated [gpu1210.oscar.ccv.brown.edu:81563] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages Client connected. ( 426.276s) [pvserver.6 ] vtkMPICommunicator.cxx:68 WARN| MPI had an error ------------------------------------------------ MPI_ERR_ARG: invalid argument of some other kind ------------------------------------------------ ( 426.276s) [pvserver.2 ] vtkMPICommunicator.cxx:68 WARN| MPI had an error ------------------------------------------------ MPI_ERR_ARG: invalid argument of some other kind ------------------------------------------------ ( 426.276s) [pvserver.4 ] vtkMPICommunicator.cxx:68 WARN| MPI had an error ------------------------------------------------ MPI_ERR_ARG: invalid argument of some other kind ------------------------------------------------ ( 426.275s) [pvserver.1 ] vtkMPICommunicator.cxx:68 WARN| MPI had an error ------------------------------------------------ MPI_ERR_ARG: invalid argument of some other kind ------------------------------------------------ ( 426.275s) [pvserver.3 ] vtkMPICommunicator.cxx:68 WARN| MPI had an error ------------------------------------------------ MPI_ERR_ARG: invalid argument of some other kind ------------------------------------------------ ( 426.275s) [pvserver.5 ] vtkMPICommunicator.cxx:68 WARN| MPI had an error ------------------------------------------------ MPI_ERR_ARG: invalid argument of some other kind ------------------------------------------------ ( 426.275s) [pvserver.7 ] vtkMPICommunicator.cxx:68 WARN| MPI had an error ------------------------------------------------ MPI_ERR_ARG: invalid argument of some other kind ------------------------------------------------ -------------------------------------------------------------------------- MPI_ABORT was invoked on rank 6 in communicator MPI_COMM_WORLD with errorcode 13. NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes. You may or may not see output from other processes, depending on exactly when Open MPI kills them. -------------------------------------------------------------------------- [gpu1210.oscar.ccv.brown.edu:81563] PMIX ERROR: UNREACHABLE in file server/pmix_server.c at line 2193  Loguru caught a signal: SIGTERM Stack trace: 43 0x4016e7 /gpfs/runtime/opt/paraview/5.8.0_release/bin/pvserver() [0x4016e7] 42 0x7efd3ce55b35 __libc_start_main + 245 41 0x401652 /gpfs/runtime/opt/paraview/5.8.0_release/bin/pvserver() [0x401652] 40 0x7efd3b0d6bcd vtkTCPNetworkAccessManager::ProcessEventsInternal(unsigned long, bool) + 541 39 0x7efd39b194d2 vtkMultiProcessController::ProcessRMIs(int, int) + 290 38 0x7efd39b18ea0 vtkMultiProcessController::ProcessRMI(int, void*, int, int) + 336 37 0x7efd3b886c3e vtkPVSessionServer::OnClientServerMessageRMI(void*, int) + 270 36 0x7efd3b87d924 vtkPVSessionBase::ExecuteStream(unsigned int, vtkClientServerStream const&, bool) + 52 35 0x7efd3b87e6be vtkPVSessionCore::ExecuteStream(unsigned int, vtkClientServerStream const&, bool) + 62 34 0x7efd3b87e889 vtkPVSessionCore::ExecuteStreamInternal(vtkClientServerStream const&, bool) + 249 33 0x7efd3a6d4e5d vtkClientServerInterpreter::ProcessStream(vtkClientServerStream const&) + 29 32 0x7efd3a6d4a76 vtkClientServerInterpreter::ProcessOneMessage(vtkClientServerStream const&, int) + 998 31 0x7efd3a6d401a vtkClientServerInterpreter::ProcessCommandInvoke(vtkClientServerStream const&, int) + 330 30 0x7efd3a6d3d6b vtkClientServerInterpreter::CallCommandFunction(char const*, vtkObjectBase*, char const*, vtkClientServerStream const&, vtkClientServerStream&) + 331 29 0x7efd3c190b6b vtkSIMetaReaderProxyCommand(vtkClientServerInterpreter*, vtkObjectBase*, char const*, vtkClientServerStream const&, vtkClientServerStream&, void*) + 1003 28 0x7efd3a6d3d6b vtkClientServerInterpreter::CallCommandFunction(char const*, vtkObjectBase*, char const*, vtkClientServerStream const&, vtkClientServerStream&) + 331 27 0x7efd3c194398 vtkSISourceProxyCommand(vtkClientServerInterpreter*, vtkObjectBase*, char const*, vtkClientServerStream const&, vtkClientServerStream&, void*) + 1464 26 0x7efd3b8b4200 vtkSISourceProxy::UpdatePipeline(int, double, bool) + 432 25 0x7efd398c5230 vtkStreamingDemandDrivenPipeline::Update(int, vtkInformationVector*) + 304 24 0x7efd398991fb vtkDemandDrivenPipeline::UpdateData(int) + 139 23 0x7efd398c68d1 vtkStreamingDemandDrivenPipeline::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 721 22 0x7efd3989a310 vtkDemandDrivenPipeline::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 496 21 0x7efd398943ea vtkCompositeDataPipeline::ForwardUpstream(vtkInformation*) + 330 20 0x7efd398c68d1 vtkStreamingDemandDrivenPipeline::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 721 19 0x7efd3989a374 vtkDemandDrivenPipeline::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 596 18 0x7efd398953b1 vtkCompositeDataPipeline::ExecuteData(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 113 17 0x7efd39897f17 vtkDemandDrivenPipeline::ExecuteData(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 55 16 0x7efd3989d410 vtkExecutive::CallAlgorithm(vtkInformation*, int, vtkInformationVector**, vtkInformationVector*) + 64 15 0x7efd3353a4a7 vtkFileSeriesReader::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 215 14 0x7efd3353a377 vtkFileSeriesReader::RequestData(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 103 13 0x7efd398a4e3e vtkImageAlgorithm::RequestData(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 110 12 0x7efd2f00abc4 vtkMPIImageReader::ExecuteDataWithInformation(vtkDataObject*, vtkInformation*) + 4948 11 0x7efd2f009141 vtkMPIImageReader::ReadSlice(int, int const*, void*) + 225 10 0x7efd2f008c19 vtkMPIImageReader::SetupFileView(vtkMPIOpaqueFileHandle&, int const*) + 393 9 0x7efd256e2ff8 MPI_File_set_view + 264 8 0x7ef9c59aca11 mca_io_ompio_file_set_view + 113 7 0x7ef9c57a350e mca_common_ompio_set_view + 990 6 Loguru caught a signal: SIGTERM Stack trace: 42 0x4016e7 /gpfs/runtime/opt/paraview/5.8.0_release/bin/pvserver() [0x4016e7] 41 0x7fe307925b35 __libc_start_main + 245 40 0x401675 /gpfs/runtime/opt/paraview/5.8.0_release/bin/pvserver() [0x401675] 39 0x7fe3045e9625 vtkMultiProcessController::ProcessRMIs(int, int) + 629 38 0x7fe3045e9049 vtkMultiProcessController::BroadcastProcessRMIs(int, int) + 121 37 0x7fe3045e8ea0 vtkMultiProcessController::ProcessRMI(int, void*, int, int) + 336 36 0x7fe30634eac5 vtkPVSessionCore::ExecuteStreamSatelliteCallback() + 165 35 0x7fe30634e889 vtkPVSessionCore::ExecuteStreamInternal(vtkClientServerStream const&, bool) + 249 34 0x7fe3051a4e5d vtkClientServerInterpreter::ProcessStream(vtkClientServerStream const&) + 29 33 0x7fe3051a4a76 vtkClientServerInterpreter::ProcessOneMessage(vtkClientServerStream const&, int) + 998 32 0x7fe3051a401a vtkClientServerInterpreter::ProcessCommandInvoke(vtkClientServerStream const&, int) + 330 31 0x7fe3051a3d6b vtkClientServerInterpreter::CallCommandFunction(char const*, vtkObjectBase*, char const*, vtkClientServerStream const&, vtkClientServerStream&) + 331 30 0x7fe306c60b6b vtkSIMetaReaderProxyCommand(vtkClientServerInterpreter*, vtkObjectBase*, char const*, vtkClientServerStream const&, vtkClientServerStream&, void*) + 1003 29 0x7fe3051a3d6b vtkClientServerInterpreter::CallCommandFunction(char const*, vtkObjectBase*, char const*, vtkClientServerStream const&, vtkClientServerStream&) + 331 28 0x7fe306c64398 vtkSISourceProxyCommand(vtkClientServerInterpreter*, vtkObjectBase*, char const*, vtkClientServerStream const&, vtkClientServerStream&, void*) + 1464 27 0x7fe306384200 vtkSISourceProxy::UpdatePipeline(int, double, bool) + 432 26 0x7fe304395230 vtkStreamingDemandDrivenPipeline::Update(int, vtkInformationVector*) + 304 25 0x7fe3043691fb vtkDemandDrivenPipeline::UpdateData(int) + 139 24 0x7fe3043968d1 vtkStreamingDemandDrivenPipeline::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 721 23 0x7fe30436a310 vtkDemandDrivenPipeline::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 496 22 0x7fe3043643ea vtkCompositeDataPipeline::ForwardUpstream(vtkInformation*) + 330 21 0x7fe3043968d1 vtkStreamingDemandDrivenPipeline::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 721 20 0x7fe30436a374 vtkDemandDrivenPipeline::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 596 19 0x7fe3043653b1 vtkCompositeDataPipeline::ExecuteData(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 113 18 0x7fe304367f17 vtkDemandDrivenPipeline::ExecuteData(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 55 17 0x7fe30436d410 vtkExecutive::CallAlgorithm(vtkInformation*, int, vtkInformationVector**, vtkInformationVector*) + 64 16 0x7fe2fe00a4a7 vtkFileSeriesReader::ProcessRequest(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 215 15 0x7fe2fe00a377 vtkFileSeriesReader::RequestData(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 103 14 0x7fe304374e3e vtkImageAlgorithm::RequestData(vtkInformation*, vtkInformationVector**, vtkInformationVector*) + 110 13 0x7fe2f9adabc4 vtkMPIImageReader::ExecuteDataWithInformation(vtkDataObject*, vtkInformation*) + 4948 12 0x7fe2f9ad9141 vtkMPIImageReader::ReadSlice(int, int const*, void*) + 225 11 0x7fe2f9ad8b8a vtkMPIImageReader::SetupFileView(vtkMPIOpaqueFileHandle&, int const*) + 250 10 0x7fe2f01cc4c5 PMPI_Type_create_subarray + 517 9 0x7fe2f017b7a5 ompi_errhandler_invoke + 277 8 0x7fe2f0469c72 /gpfs/rt/7.2/opt/paraview/5.8.0_release/bin/../lib64/libvtkParallelMPI-pv5.8.so.1(+0x7c72) [0x7fe2f0469c72] 7 0x7fe2f018d24c ompi_mpi_abort + 716 6 0x7fe2a0bb249b /gpfs/runtime/opt/mpi/openmpi_4.0.4_gcc/lib/openmpi/mca_errmgr_default_app.so(+0x149b) [0x7fe 0x7efd257159ab ompi_coll_base_allreduce_intra_recursivedoubling + 667 5 0x7efd257151f3 ompi_coll_base_sendrecv_actual + 163 4 0x7efd256bc375 ompi_request_default_wait + 261 3 0x7efd1c60ffec opal_progress + 44 2 0x7efccfdcfe6a ucp_worker_progress + 106 1 0x7efccf6be65b uct_rc_mlx5_iface_progress + 571 0 0x7efd3ce69250 /lib64/libc.so.6(+0x35250) [0x7efd3ce69250] ( 427.328s) [pvserver.0 ] :0 FATL| Signal: SIGTERM 2a0bb249b] 5 0x7fe2e74057a6 orte_errmgr_base_abort + 262 4 0x7fe2e4a5bd76 /gpfs/runtime/opt/mpi/openmpi_4.0.4_gcc/lib/openmpi/mca_ess_pmi.so(+0x1d76) [0x7fe2e4a5bd76] 3 0x7fe2c5ee24b7 pmix3x_abort + 183 2 0x7fe2c5f3ad8d OPAL_MCA_PMIX3X_PMIx_Abort + 1501 1 0x7fe3022186d5 pthread_cond_wait + 197 0 0x7fe307939250 /lib64/libc.so.6(+0x35250) [0x7fe307939250] ( 427.328s) [pvserver.2 ] :0 FATL| Signal: SIGTERM [gpu1210.oscar.ccv.brown.edu:81563] 6 more processes have sent help message help-mpi-api.txt / mpi-abort PServer Initialization completed Job completed