[CAI-63] Chatbot: create vector index from pipeline

.changeset/wild-wasps-wink.md

@@ -0,0 +1,5 @@
+---
+"infrastructure": patch
+---
+
+Added chatbot env vars to codebuild project

.github/actions/chatbot/action.yaml

@@ -0,0 +1,44 @@
+name: chatbot
+description: Recreate Chatbot LLamaindex Vector Index
+
+inputs:
+  aws_region:
+    description: 'The AWS region selected'
+    required: false
+    default: 'eu-south-1'
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - name: Setup Python
+      uses: kishaningithub/setup-python-amazon-linux@282d998c8f2c4678ac4f872e4dc3b7d167b59fa7 # v1
+      with:
+        python-version: '3.12'
+
+    - name: Install and configure Poetry
+      uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1
+      with:
+        version: 1.8.3
+        virtualenvs-create: false
+        installer-parallel: true
+
+    - name: Install python libraries
+      shell: bash
+      run: |
+        cd apps/chatbot
+        poetry install --no-interaction
+
+#    - name: Run tests
+#      run: |
+#        cd apps/chatbot
+#        poetry run pytest
+#        coverage report
+#
+
+    - name: Create Vector Index
+      shell: bash
+      run: |
+        cd apps/chatbot
+        poetry run python src/modules/create_vector_index.py --params config/params.yaml

.github/workflows/chatbot_reindex.yaml

@@ -0,0 +1,118 @@
+name: Chatbot Reindex
+
+on:
+  pull_request:
+    types:
+      - closed # Trigger when the PR is closed
+    branches:
+      - 'main' # Target branch is main
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'The environment used as target'
+        type: choice
+        required: true
+        default: dev
+        options:
+          - dev
+          - prod
+
+  # Allows external webhook trigger
+  repository_dispatch:
+    types:
+      - webhook
+
+# These permissions are needed to interact with GitHub's OIDC Token endpoint.
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+
+  cd_deploy:
+    name: Llama reindex (on ${{ matrix.environment }})
+    # Trigger when the PR is merged or on a webhook (e.g. trigger deploy from CMS)
+    if: (github.event_name == 'pull_request' && github.event.pull_request.merged == true) || github.event_name == 'repository_dispatch'
+    runs-on: codebuild-${{ matrix.environment }}-github-runner-${{ github.run_id }}-${{ github.run_attempt }}
+    strategy:
+      matrix:
+        environment: [ 'dev' ]
+      fail-fast: false # Do not cancel any other jobs in the matrix, if a single job fails
+
+    environment: ${{ matrix.environment }}
+
+    concurrency:
+      group: ${{ github.workflow }}-${{ matrix.environment }}
+      cancel-in-progress: false
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
+
+      - name: Increase action disk space
+        uses: ./.github/actions/increase-disk-space
+
+      - name: Build NextJS Website
+        uses: ./.github/actions/build-nextjs-website
+        with:
+          path_to_gitbook_docs: ${{ github.workspace }}/apps/nextjs-website/docs
+          cookie_domain_script: ${{ secrets.COOKIE_DOMAIN_SCRIPT }}
+          allow_crawler: ${{ vars.ALLOW_CRAWLER }}
+          cognito_user_pool_id: ${{ secrets.NEXT_PUBLIC_COGNITO_USER_POOL_ID }}
+          cognito_identity_pool_id: ${{ secrets.NEXT_PUBLIC_COGNITO_IDENTITY_POOL_ID }}
+          cognito_app_client_id: ${{ secrets.NEXT_PUBLIC_COGNITO_USER_POOL_WEB_CLIENT_ID }}
+          chatbot_host: ${{ vars.NEXT_PUBLIC_CHATBOT_HOST }}
+          chat_max_history_messages: ${{ vars.NEXT_PUBLIC_CHAT_MAX_HISTORY_MESSAGES }}
+          website_name: ${{ vars.NEXT_PUBLIC_WEBSITE_NAME }}
+          organization_name: ${{ vars.NEXT_PUBLIC_ORGANIZATION_NAME }}
+          organization_logo: ${{ vars.NEXT_PUBLIC_ORGANIZATION_LOGO }}
+          organization_social_links: ${{ vars.NEXT_PUBLIC_ORGANIZATION_SOCIAL_LINKS }}
+          chatbot_active: ${{ vars.NEXT_PUBLIC_CHATBOT_ACTIVE }}
+          cognito_aws_region: 'eu-south-1'
+          strapi_endpoint: ${{ vars.STRAPI_ENDPOINT }}
+          strapi_api_token: ${{ secrets.STRAPI_API_TOKEN }}
+          fetch_from_strapi: ${{ vars.FETCH_FROM_STRAPI }}
+
+      - name: Chatbot Vector Index
+        uses: ./.github/actions/chatbot
+
+  manual_deploy:
+    name: Llama reindex (manual on ${{ inputs.environment }})
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: codebuild-${{ inputs.environment }}-github-runner-${{ github.run_id }}-${{ github.run_attempt }}
+    environment: ${{ inputs.environment }}
+
+    concurrency:
+      group: ${{ github.workflow }}-${{ inputs.environment }}
+      cancel-in-progress: false
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
+
+      - name: Increase action disk space
+        uses: ./.github/actions/increase-disk-space
+
+      - name: Build NextJS Website
+        uses: ./.github/actions/build-nextjs-website
+        with:
+          path_to_gitbook_docs: ${{ github.workspace }}/apps/nextjs-website/docs
+          cookie_domain_script: ${{ secrets.COOKIE_DOMAIN_SCRIPT }}
+          allow_crawler: ${{ vars.ALLOW_CRAWLER }}
+          cognito_user_pool_id: ${{ secrets.NEXT_PUBLIC_COGNITO_USER_POOL_ID }}
+          cognito_identity_pool_id: ${{ secrets.NEXT_PUBLIC_COGNITO_IDENTITY_POOL_ID }}
+          cognito_app_client_id: ${{ secrets.NEXT_PUBLIC_COGNITO_USER_POOL_WEB_CLIENT_ID }}
+          chatbot_host: ${{ vars.NEXT_PUBLIC_CHATBOT_HOST }}
+          chat_max_history_messages: ${{ vars.NEXT_PUBLIC_CHAT_MAX_HISTORY_MESSAGES }}
+          website_name: ${{ vars.NEXT_PUBLIC_WEBSITE_NAME }}
+          organization_name: ${{ vars.NEXT_PUBLIC_ORGANIZATION_NAME }}
+          organization_logo: ${{ vars.NEXT_PUBLIC_ORGANIZATION_LOGO }}
+          organization_social_links: ${{ vars.NEXT_PUBLIC_ORGANIZATION_SOCIAL_LINKS }}
+          chatbot_active: ${{ vars.NEXT_PUBLIC_CHATBOT_ACTIVE }}
+          cognito_aws_region: 'eu-south-1'
+          strapi_endpoint: ${{ vars.STRAPI_ENDPOINT }}
+          strapi_api_token: ${{ secrets.STRAPI_API_TOKEN }}
+          fetch_from_strapi: ${{ vars.FETCH_FROM_STRAPI }}
+
+      - name: Chatbot Vector Index
+        uses: ./.github/actions/chatbot

.github/workflows/deploy_website.yaml

@@ -33,7 +33,7 @@ jobs:
     name: Deploy to S3 (on ${{ matrix.environment }})
     # Trigger when the PR is merged or on a webhook (e.g. trigger deploy from CMS)
     if: (github.event_name == 'pull_request' && github.event.pull_request.merged == true) || github.event_name == 'repository_dispatch'
-    runs-on: 'ubuntu-latest'
+    runs-on: codebuild-${{ matrix.environment }}-github-runner-${{ github.run_id }}-${{ github.run_attempt }}
     strategy:
       matrix:
         environment: [ 'dev' ]
@@ -76,10 +76,13 @@ jobs:
           strapi_api_token: ${{ secrets.STRAPI_API_TOKEN }}
           fetch_from_strapi: ${{ vars.FETCH_FROM_STRAPI }}
 
+      - name: Chatbot Vector Index
+        uses: ./.github/actions/chatbot
+
   manual_deploy:
     name: Deploy to S3 (manual on ${{ inputs.environment }})
     if: github.event_name == 'workflow_dispatch'
-    runs-on: 'ubuntu-latest'
+    runs-on: codebuild-${{ inputs.environment }}-github-runner-${{ github.run_id }}-${{ github.run_attempt }}
     environment: ${{ inputs.environment }}
 
     concurrency:
@@ -116,3 +119,6 @@ jobs:
           strapi_endpoint: ${{ vars.STRAPI_ENDPOINT }}
           strapi_api_token: ${{ secrets.STRAPI_API_TOKEN }}
           fetch_from_strapi: ${{ vars.FETCH_FROM_STRAPI }}
+
+      - name: Chatbot Vector Index
+        uses: ./.github/actions/chatbot

.github/workflows/deploy_website_content.yaml

@@ -65,3 +65,7 @@ jobs:
           strapi_endpoint: ${{ vars.STRAPI_ENDPOINT }}
           strapi_api_token: ${{ secrets.STRAPI_API_TOKEN }}
           fetch_from_strapi: ${{ vars.FETCH_FROM_STRAPI }}
+
+      - name: Chatbot Vector Index
+        uses: ./.github/actions/chatbot
+        runs-on: codebuild-${{ inputs.environment }}-github-runner-${{ github.run_id }}-${{ github.run_attempt }}

apps/infrastructure/src/main.tf

@@ -149,6 +149,8 @@ module "cicd" {
 
   website_bucket = module.website.website_bucket
   website_cdn    = module.website.website_cdn
+
+  chatbot_env_vars = var.create_chatbot ? module.chatbot[0].lambda_env_variables : {}
 }
 
 module "active_campaign" {

apps/infrastructure/src/modules/chatbot/outputs.tf

@@ -102,6 +102,11 @@ output "lambda_role_name" {
   value       = module.lambda_function.lambda_role_name
 }
 
+output "lambda_env_variables" {
+  description = "Environment variables of the Lambda Function"
+  value       = local.lambda_env_variables
+}
+
 # CloudWatch Log Group
 output "lambda_cloudwatch_log_group_arn" {
   description = "The ARN of the Cloudwatch Log Group"

apps/infrastructure/src/modules/cicd/codebuild.tf

@@ -13,6 +13,15 @@ resource "aws_codebuild_project" "github_runner" {
     type                        = "LINUX_CONTAINER"
     image_pull_credentials_type = "CODEBUILD"
     privileged_mode             = true
+
+    dynamic "environment_variable" {
+      for_each = var.chatbot_env_vars
+
+      content {
+        name  = environment_variable.key
+        value = environment_variable.value
+      }
+    }
   }
 
   source {

apps/infrastructure/src/modules/cicd/variables.tf

@@ -79,3 +79,9 @@ variable "create_chatbot" {
   description = "Defines if chatbot should be created"
   default     = false
 }
+
+variable "chatbot_env_vars" {
+  type        = map(string)
+  description = "Chatbot environment variables"
+  default     = {}
+}