NPE after force infrastructure restart

[mmorhun]

Description

After force reboot of host system, recovering of previously running workspace is failed with NPE.
Che server logs:

2019-05-11 06:19:51,316[main]             [INFO ] [o.a.catalina.startup.Catalina 700]   - Server startup in 13438 ms
2019-05-11 06:20:52,465[nio-8080-exec-7]  [INFO ] [o.e.c.a.w.s.WorkspaceRuntimes 273]   - Runtime for workspace 'workspacesyftnj2jpzap3c9c' is requested but there is no cached one. Recovering it.
2019-05-11 06:20:52,471[nio-8080-exec-7]  [ERROR] [c.a.c.r.RuntimeExceptionMapper 47]   - Internal Server Error occurred, error time: 2019-05-11 06:20:52
java.lang.NullPointerException: null
	at org.eclipse.che.api.workspace.server.WorkspaceRuntimes.recoverOne(WorkspaceRuntimes.java:589)
	at org.eclipse.che.api.workspace.server.WorkspaceRuntimes.getInternalRuntime(WorkspaceRuntimes.java:276)
	at org.eclipse.che.api.workspace.server.WorkspaceRuntimes.injectRuntime(WorkspaceRuntimes.java:224)
	at org.eclipse.che.api.workspace.server.WorkspaceManager.normalizeState(WorkspaceManager.java:534)
	at org.eclipse.che.api.workspace.server.WorkspaceManager.getWorkspace(WorkspaceManager.java:174)
	at org.eclipse.che.api.workspace.server.WorkspaceService.getByKey(WorkspaceService.java:290)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.everrest.core.impl.method.DefaultMethodInvoker.invokeMethod(DefaultMethodInvoker.java:140)
	at org.everrest.core.impl.method.DefaultMethodInvoker.invokeMethod(DefaultMethodInvoker.java:60)
	at org.everrest.core.impl.RequestDispatcher.doInvokeResource(RequestDispatcher.java:306)
	at org.everrest.core.impl.RequestDispatcher.invokeSubResourceMethod(RequestDispatcher.java:297)
	at org.everrest.core.impl.RequestDispatcher.dispatch(RequestDispatcher.java:233)
	at org.everrest.core.impl.RequestDispatcher.dispatch(RequestDispatcher.java:128)
	at org.everrest.core.impl.RequestHandlerImpl.handleRequest(RequestHandlerImpl.java:62)
	at org.everrest.core.impl.EverrestProcessor.process(EverrestProcessor.java:120)
	at org.everrest.core.servlet.EverrestServlet.service(EverrestServlet.java:61)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
	at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:290)
	at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:280)
	at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:184)
	at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:89)
	at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85)
	at org.eclipse.che.api.local.filters.EnvironmentInitializationFilter.doFilter(EnvironmentInitializationFilter.java:64)
	at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
	at org.eclipse.che.commons.logback.filter.RequestIdLoggerFilter.doFilter(RequestIdLoggerFilter.java:50)
	at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
	at org.apache.catalina.filters.CorsFilter.handleNonCORS(CorsFilter.java:364)
	at org.apache.catalina.filters.CorsFilter.doFilter(CorsFilter.java:170)
	at org.eclipse.che.api.core.cors.CheCorsFilter.doFilter(CheCorsFilter.java:58)
	at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
	at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:121)
	at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:133)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
	at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:493)
	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81)
	at ch.qos.logback.access.tomcat.LogbackValve.invoke(LogbackValve.java:256)
	at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:685)
	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342)
	at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:800)
	at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
	at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:806)
	at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1498)
	at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
	at java.lang.Thread.run(Thread.java:748)

Also after that Workspaces tab in dashboard is inoperable with error message: Failed to retrieve workspace MyWorkspace data. Reason: Internal Server Error occurred, error time: 2019-05-11 06:52:33. Moreover it is not possible to fix the situation using API calls (like try to stop starting workspace).

Reproduction Steps

Workspace was created from devfile:

{
  "specVersion": "0.0.1",
  "name": "multiple-dockerimages",
  "projects": [
    {
      "name": "nodejs-mongo-app",
      "source": {
        "type": "git",
        "location": "/~https://github.com/ijason/NodeJS-Sample-App.git"
      }
    }
  ],
  "components": [
    {
      "name": "theia-editor",
      "type": "cheEditor",
      "id": "eclipse/che-theia/next"
    },
    {
      "name": "exec-plugin",
      "type": "chePlugin",
      "id": "eclipse/che-machine-exec-plugin/0.0.1"
    },  
    {
      "name": "mongodb",
      "type": "dockerimage",
      "image": "a/b",
      "endpoints": [
        {
          "name": "mongo",
          "port": 27017,
          "attributes": {
            "public": "false",
            "discoverable": "true"
          }
        }
      ],
      "mountSources": true,
      "volumes": [
        {
          "name": "mongo-storage",
          "containerPath": "/data/db"
        }
      ],
      "memoryLimit": "2500Mi"
    },
    {
      "name": "nodejs-app",
      "type": "openshift",
      "reference": "node-js.yaml",
      "referenceContent": "apiVersion: v1\nkind: List\nitems:\n-\n  apiVersion: apps/v1\n  kind: Deployment\n  metadata:\n    name: web\n    labels:\n      app: nodejs\n  spec:\n    replicas: 2\n    selector:\n      name: web\n    template:\n      metadata:\n        labels:\n          app: nodejs\n        name: web-controller\n      spec:\n        containers:\n        - image: node:0.10.40\n          command: ['tail', '-f', '/dev/null']\n          args: []\n          name: web\n          ports:\n          - containerPort: 3000\n            name: http-server\n          volumeMounts:\n           - mountPath: /projects\n             name: projects\n        volumes:\n         - name: projects\n           persistentVolumeClaim:\n             claimName: projects\n- apiVersion: v1\n  kind: PersistentVolumeClaim\n  metadata:\n    name: projects\n  spec:\n    accessModes:\n     - ReadWriteOnce\n    resources:\n      requests:\n        storage: 2Gi\n-\n  apiVersion: v1\n  kind: Service\n  metadata:\n    name: web\n    labels:\n      name: web\n  spec:\n    type: LoadBalancer\n    ports:\n      - name: web\n        port: 80\n        targetPort: 3000\n        protocol: TCP\n    selector:\n      app: nodejs\n- apiVersion: v1\n  kind: Route\n  metadata:\n    name: che\n  spec:\n    to:\n      kind: Service\n      name: web\n    port:\n      targetPort: web\n"
    }
  ],
  "commands": [
    {
      "name": "run",
      "actions": [
        {
          "type": "exec",
          "component": "nodejs-app",
          "command": "cd ${CHE_PROJECTS_ROOT}/nodejs-mongo-app/EmployeeDB/ && npm install && node app.js"
        }
      ]
    }
  ]
}

Where a/b is tagged eclipse/che-theia-dev:nightly

I've got it on host OS reboot with running workspace inside minishift

[mmorhun]

Rescaling of Che master didn't help :(