feat: Collect k8s events (#465)

* Collect k8s events

Signed-off-by: Anatoliy Bazko <abazko@redhat.com>

README.md

@@ -422,7 +422,7 @@ _See code: [src/commands/workspace/list.ts](/~https://github.com/che-incubator/che
 
 ## `chectl workspace:logs`
 
-Collect workspace logs
+Collect workspace(s) logs
 
 ```
 USAGE
@@ -432,12 +432,11 @@ OPTIONS
   -d, --directory=directory                Directory to store logs into
   -h, --help                               show CLI help
 
-  -n, --chenamespace=chenamespace          [default: che] Kubernetes namespace where Che server is supposed to be
-                                           deployed
-
-  -w, --workspace=workspace                Target workspace. Can be omitted if only one Workspace is running
+  -n, --namespace=namespace                (required) The namespace where workspace is located. Can be found in
+                                           workspace configuration 'attributes.infrastructureNamespace' field.
 
-  --follow                                 Follow workspace creation logs
+  -w, --workspace=workspace                (required) Target workspace id. Can be found in workspace configuration 'id'
+                                           field.
 
   --listr-renderer=default|silent|verbose  [default: default] Listr renderer
 ```

src/api/che.ts

@@ -9,10 +9,13 @@
  **********************************************************************/
 import { CoreV1Api, KubeConfig } from '@kubernetes/client-node'
 import axios from 'axios'
+import * as cp from 'child_process'
 import { cli } from 'cli-ux'
+import * as commandExists from 'command-exists'
 import * as fs from 'fs-extra'
 import * as yaml from 'js-yaml'
 import * as path from 'path'
+import { setInterval } from 'timers'
 
 import { OpenShiftHelper } from '../api/openshift'
 
@@ -303,79 +306,89 @@ export class CheHelper {
   }
 
   /**
-   * Reads logs from all new pods that starting to run.
-   * It basically lists existed pods and starts following logs from a new ones.
+   * Finds workspace pods and reads logs from it.
    */
-  async readAllNewPodLog(namespace: string, directory: string): Promise<void> {
-    const processedPods = new Set<string>((await this.kube.listNamespacedPod(namespace)).items.map(pod => pod.metadata!.name!))
-    setInterval(async () => this.readPodLogBySelectorIgnoreProcessed(namespace, undefined, directory, processedPods, true), CheHelper.POLL_INTERVAL)
+  async readWorkspacePodLog(namespace: string, workspaceId: string, directory: string): Promise<boolean> {
+    const podLabelSelector = `che.workspace_id=${workspaceId}`
+
+    let workspaceIsRun = false
+
+    const pods = await this.kube.listNamespacedPod(namespace, undefined, podLabelSelector)
+    if (pods.items.length) {
+      workspaceIsRun = true
+    }
+
+    for (const pod of pods.items) {
+      for (const containerStatus of pod.status!.containerStatuses!) {
+        workspaceIsRun = workspaceIsRun && !!containerStatus.state && !!containerStatus.state.running
+      }
+    }
+
+    const follow = !workspaceIsRun
+    await this.readPodLog(namespace, podLabelSelector, directory, follow)
+    await this.readNamespaceEvents(namespace, directory, follow)
+
+    return workspaceIsRun
   }
 
   /**
    * Reads logs from pods that match a given selector.
    */
-  async readPodLogBySelector(namespace: string, selector: string | undefined, directory: string, follow: boolean): Promise<void> {
-    const processedPods = new Set<string>()
+  async readPodLog(namespace: string, podLabelSelector: string | undefined, directory: string, follow: boolean): Promise<void> {
+    const processedContainers = new Map<string, Set<string>>()
     if (follow) {
-      setInterval(async () => this.readPodLogBySelectorIgnoreProcessed(namespace, selector, directory, processedPods, follow), CheHelper.POLL_INTERVAL)
+      setInterval(async () => this.readContainerLogIgnoreProcessed(namespace, podLabelSelector, directory, processedContainers, follow), CheHelper.POLL_INTERVAL)
     } else {
-      await this.readPodLogBySelectorIgnoreProcessed(namespace, selector, directory, processedPods, follow)
+      await this.readContainerLogIgnoreProcessed(namespace, podLabelSelector, directory, processedContainers, follow)
     }
   }
 
   /**
-   * Reads logs from pods that match a given selector with exception of already processed ones.
-   * Once log is read the pod is marked as processed.
+   * Reads containers logs inside pod that match a given selector.
    */
-  async readPodLogBySelectorIgnoreProcessed(namespace: string, selector: string | undefined, directory: string, processedPods: Set<string>, follow: boolean): Promise<void> {
-    const pods = await this.kube.listNamespacedPod(namespace, selector)
+  async readContainerLogIgnoreProcessed(namespace: string, podLabelSelector: string | undefined, directory: string, processedContainers: Map<string, Set<string>>, follow: boolean): Promise<void> {
+    const pods = await this.kube.listNamespacedPod(namespace, undefined, podLabelSelector)
 
     for (const pod of pods.items) {
       const podName = pod.metadata!.name!
+      if (!processedContainers.has(podName)) {
+        processedContainers.set(podName, new Set<string>())
+      }
 
-      if (!processedPods.has(podName)) {
-        processedPods.add(podName)
-        await this.readPodLogByName(namespace, podName, directory, follow)
+      if (!pod.status || !pod.status.containerStatuses) {
+        return
       }
-    }
-  }
 
-  /**
-   * Reads log from pod that matches a given name.
-   */
-  async readPodLogByName(namespace: string, podName: string, directory: string, follow: boolean): Promise<void> {
-    const processedContainers = new Set<string>()
-    if (follow) {
-      setInterval(async () => this.readPodLogByNameIgnoreProcessed(namespace, podName, directory, processedContainers, follow), 100)
-    } else {
-      await this.readPodLogByNameIgnoreProcessed(namespace, podName, directory, processedContainers, follow)
+      for (const containerStatus of pod.status.containerStatuses) {
+        if (!containerStatus.state || !containerStatus.state.running) {
+          continue
+        }
+
+        const containerName = containerStatus.name
+        if (!processedContainers.get(podName)!.has(containerName)) {
+          processedContainers.get(podName)!.add(containerName)
+          await this.readContainerLog(namespace, podName, containerName, directory, follow)
+        }
+      }
     }
   }
 
   /**
-   * Reads log from all containers in the pod with exception of already processed ones.
-   * Once log is read the container is marked as processed.
+   * Reads all namespace events and store into a file.
    */
-  async readPodLogByNameIgnoreProcessed(namespace: string, podName: string, directory: string, processedContainers: Set<string>, follow: boolean): Promise<void> {
-    const pod = await this.kube.readNamespacedPod(podName, namespace)
-    if (!pod) {
-      return
-    }
-
-    if (!pod.status || !pod.status.containerStatuses) {
-      return
-    }
+  async readNamespaceEvents(namespace: string, directory: string, follow: boolean): Promise<void> {
+    const fileName = path.resolve(directory, namespace, 'events.txt')
+    fs.ensureFileSync(fileName)
 
-    for (const container of pod.status.containerStatuses) {
-      if (!container.state || !container.state.running) {
-        continue
-      }
+    const cli = (commandExists.sync('kubectl') && 'kubectl') || (commandExists.sync('oc') && 'oc')
+    if (cli) {
+      const command = 'get events'
+      const namespaceParam = `-n ${namespace}`
+      const watchParam = follow && '--watch' || ''
 
-      const containerName = container.name
-      if (!processedContainers.has(containerName)) {
-        processedContainers.add(containerName)
-        await this.readContainerLog(namespace, podName, containerName, directory, follow)
-      }
+      cp.exec(`${cli} ${command} ${namespaceParam} ${watchParam} >> ${fileName}`)
+    } else {
+      throw new Error('No events are collected. \'kubectl\' or \'oc\' is required to perform the task.')
     }
   }
 
@@ -385,6 +398,7 @@ export class CheHelper {
   private async readContainerLog(namespace: string, podName: string, containerName: string, directory: string, follow: boolean): Promise<void> {
     const fileName = path.resolve(directory, namespace, podName, `${containerName}.log`)
     fs.ensureFileSync(fileName)
+
     return this.kube.readNamespacedPodLog(podName, namespace, containerName, fileName, follow)
   }
 

src/api/kube.ts

@@ -8,7 +8,7 @@
  * SPDX-License-Identifier: EPL-2.0
  **********************************************************************/
 
-import { ApiextensionsV1beta1Api, ApisApi, AppsV1Api, CoreV1Api, CustomObjectsApi, ExtensionsV1beta1Api, KubeConfig, Log, RbacAuthorizationV1Api, V1beta1CustomResourceDefinition, V1beta1IngressList, V1ClusterRole, V1ClusterRoleBinding, V1ConfigMap, V1ConfigMapEnvSource, V1Container, V1DeleteOptions, V1Deployment, V1DeploymentList, V1DeploymentSpec, V1EnvFromSource, V1LabelSelector, V1ObjectMeta, V1PersistentVolumeClaimList, V1Pod, V1PodList, V1PodSpec, V1PodTemplateSpec, V1Role, V1RoleBinding, V1RoleRef, V1Secret, V1ServiceAccount, V1ServiceList, V1Subject } from '@kubernetes/client-node'
+import { ApiextensionsV1beta1Api, ApisApi, AppsV1Api, CoreV1Api, CustomObjectsApi, ExtensionsV1beta1Api, KubeConfig, Log, RbacAuthorizationV1Api, V1beta1CustomResourceDefinition, V1beta1IngressList, V1ClusterRole, V1ClusterRoleBinding, V1ConfigMap, V1ConfigMapEnvSource, V1Container, V1DeleteOptions, V1Deployment, V1DeploymentList, V1DeploymentSpec, V1EnvFromSource, V1LabelSelector, V1NamespaceList, V1ObjectMeta, V1PersistentVolumeClaimList, V1Pod, V1PodList, V1PodSpec, V1PodTemplateSpec, V1Role, V1RoleBinding, V1RoleRef, V1Secret, V1ServiceAccount, V1ServiceList, V1Subject } from '@kubernetes/client-node'
 import { Context } from '@kubernetes/client-node/dist/config_types'
 import axios from 'axios'
 import { cli } from 'cli-ux'
@@ -1258,10 +1258,26 @@ export class KubeHelper {
     throw new Error('ERR_LIST_PVCS')
   }
 
-  async listNamespacedPod(namespace: string, selector?: string): Promise<V1PodList> {
+  async listNamespace(): Promise<V1NamespaceList> {
     const k8sApi = this.kc.makeApiClient(CoreV1Api)
     try {
-      const res = await k8sApi.listNamespacedPod(namespace, true, undefined, undefined, undefined, selector)
+      const res = await k8sApi.listNamespace()
+      if (res && res.body) {
+        return res.body
+      } else {
+        return {
+          items: []
+        }
+      }
+    } catch (e) {
+      throw this.wrapK8sClientError(e)
+    }
+  }
+
+  async listNamespacedPod(namespace: string, fieldSelector?: string, labelSelector?: string): Promise<V1PodList> {
+    const k8sApi = this.kc.makeApiClient(CoreV1Api)
+    try {
+      const res = await k8sApi.listNamespacedPod(namespace, true, undefined, undefined, fieldSelector, labelSelector)
       if (res && res.body) {
         return res.body
       } else {

src/commands/server/logs.ts

@@ -29,7 +29,8 @@ export default class Logs extends Command {
     'deployment-name': cheDeployment,
     directory: string({
       char: 'd',
-      description: 'Directory to store logs into'
+      description: 'Directory to store logs into',
+      env: 'CHE_LOGS'
     })
   }
 
@@ -44,14 +45,14 @@ export default class Logs extends Command {
     tasks.add(k8sTasks.testApiTasks(flags, this))
     tasks.add(cheTasks.verifyCheNamespaceExistsTask(flags, this))
     tasks.add(cheTasks.serverLogsTasks(flags, false))
+    tasks.add(cheTasks.namespaceEventsTask(flags.chenamespace, this, false))
 
     try {
+      this.log(`Eclipse Che logs will be available in '${ctx.directory}'`)
       await tasks.run(ctx)
       this.log('Command server:logs has completed successfully.')
     } catch (error) {
       this.error(error)
-    } finally {
-      this.log(`Eclipse Che logs will be available in '${ctx.directory}'`)
     }
 
     notifier.notify({

src/commands/server/start.ts

@@ -112,7 +112,8 @@ export default class Start extends Command {
     }),
     directory: string({
       char: 'd',
-      description: 'Directory to store logs into'
+      description: 'Directory to store logs into',
+      env: 'CHE_LOGS'
     }),
     'workspace-pvc-storage-class-name': string({
       description: 'persistent volume(s) storage class name to use to store Eclipse Che workspaces data',
@@ -235,13 +236,20 @@ export default class Start extends Command {
       task: () => new Listr(cheTasks.serverLogsTasks(flags, true))
     }], listrOptions)
 
+    const eventTasks = new Listr([{
+      title: 'Start following events',
+      task: () => new Listr(cheTasks.namespaceEventsTask(flags.chenamespace, this, true))
+    }], listrOptions)
+
     try {
       await preInstallTasks.run(ctx)
 
       if (!ctx.isCheDeployed) {
         this.checkPlatformCompatibility(flags)
         await platformCheckTasks.run(ctx)
+        this.log(`Eclipse Che logs will be available in '${ctx.directory}'`)
         await logsTasks.run(ctx)
+        await eventTasks.run(ctx)
         await installTasks.run(ctx)
       } else if (!ctx.isCheReady
         || (ctx.isPostgresDeployed && !ctx.isPostgresReady)
@@ -259,8 +267,6 @@ export default class Start extends Command {
       this.log('Command server:start has completed successfully.')
     } catch (err) {
       this.error(err)
-    } finally {
-      this.log(`Eclipse Che logs will be available in '${ctx.directory}'`)
     }
 
     notifier.notify({

src/commands/workspace/logs.ts

@@ -15,28 +15,30 @@ import * as notifier from 'node-notifier'
 import * as os from 'os'
 import * as path from 'path'
 
-import { cheNamespace, listrRenderer } from '../../common-flags'
+import { listrRenderer } from '../../common-flags'
 import { CheTasks } from '../../tasks/che'
 import { K8sTasks } from '../../tasks/platforms/k8s'
 
 export default class Logs extends Command {
-  static description = 'Collect workspace logs'
+  static description = 'Collect workspace(s) logs'
 
   static flags = {
     help: flags.help({ char: 'h' }),
-    chenamespace: cheNamespace,
     'listr-renderer': listrRenderer,
-    follow: flags.boolean({
-      description: 'Follow workspace creation logs',
-      default: false
-    }),
     workspace: string({
       char: 'w',
-      description: 'Target workspace. Can be omitted if only one Workspace is running'
+      description: 'Target workspace id. Can be found in workspace configuration \'id\' field.',
+      required: true
+    }),
+    namespace: string({
+      char: 'n',
+      description: 'The namespace where workspace is located. Can be found in workspace configuration \'attributes.infrastructureNamespace\' field.',
+      required: true
     }),
     directory: string({
       char: 'd',
-      description: 'Directory to store logs into'
+      description: 'Directory to store logs into',
+      env: 'CHE_LOGS'
     })
   }
 
@@ -49,20 +51,16 @@ export default class Logs extends Command {
 
     const tasks = new Listr([], { renderer: flags['listr-renderer'] as any })
     tasks.add(k8sTasks.testApiTasks(flags, this))
-    tasks.add(cheTasks.verifyCheNamespaceExistsTask(flags, this))
-    if (!flags.follow) {
-      tasks.add(cheTasks.verifyWorkspaceRunTask(flags, this))
-    }
-    tasks.add(cheTasks.workspaceLogsTasks(flags))
+    tasks.add(cheTasks.workspaceLogsTasks(flags.namespace, flags.workspace))
 
     try {
+      this.log(`Eclipse Che logs will be available in '${ctx.directory}'`)
       await tasks.run(ctx)
 
-      if (flags.follow) {
-        this.log(`chectl is still running and keeps collecting logs in '${ctx.directory}'`)
-      } else {
-        this.log(`Workspace logs is available in '${ctx.directory}'`)
-        this.log('Command workspace:logs has completed successfully.')
+      if (!ctx['workspace-run']) {
+        this.log(`Workspace ${flags.workspace} probably hasn't been started yet.`)
+        this.log('The program will keep running and collecting logs...')
+        this.log('Terminate the program when all logs are gathered...')
       }
     } catch (error) {
       this.error(error)