op-atlas as a project/artifact source

warehouse/metrics_mesh/macros/onchain/chain_id_to_chain_name.py

@@ -0,0 +1,74 @@
+from sqlglot import expressions as exp
+from sqlmesh import macro
+from sqlmesh.core.macros import MacroEvaluator
+
+
+@macro()
+def chain_id_to_chain_name(
+    evaluator: MacroEvaluator,
+    chain_id: exp.Expression,
+):
+    """
+    Macro to translate chain_id to chain_name
+    Note: we need to keep this synced with
+    /~https://github.com/voteagora/op-atlas/blob/main/app/src/lib/oso.ts#L6
+    /~https://github.com/opensource-observer/oss-directory/blob/main/src/resources/schema/blockchain-address.json
+    """
+    chain_name = exp.Case(
+        ifs=[
+            exp.If(
+                this=exp.EQ(
+                    this=chain_id,
+                    expression=exp.Literal.number(1),
+                ),
+                true=exp.Literal.string("mainnet"),
+            ),
+        ],
+        default=exp.cast(chain_id, "string"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(10)),
+        exp.Literal.string("optimism"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(8453)),
+        exp.Literal.string("base"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(34443)),
+        exp.Literal.string("mode"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(480)),
+        exp.Literal.string("worldchain"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(8008)),
+        exp.Literal.string("polynomial"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(60808)),
+        exp.Literal.string("bob"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(57073)),
+        exp.Literal.string("ink"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(1135)),
+        exp.Literal.string("lisk"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(1750)),
+        exp.Literal.string("metal"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(185)),
+        exp.Literal.string("mint"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(6805)),
+        exp.Literal.string("race"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(360)),
+        exp.Literal.string("shape"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(1868)),
+        exp.Literal.string("soneium"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(1923)),
+        exp.Literal.string("swell"),
+    ).when(
+        exp.EQ(this=chain_id, expression=exp.Literal.number(7777777)),
+        exp.Literal.string("zora"),
+    )
+    return exp.Upper(this=chain_name)

warehouse/metrics_mesh/macros/url_parts.py

@@ -0,0 +1,31 @@
+from sqlglot import expressions as exp
+from sqlmesh import macro
+from sqlmesh.core.macros import MacroEvaluator
+
+
+@macro()
+def url_parts(
+    evaluator: MacroEvaluator,
+    url: exp.ExpOrStr,
+    index: int
+) -> exp.Expression:
+    """
+    This will parse a URI/URL, split on '/' and return the part
+    - 1 is usually the domain
+    - 2 is usually the first part of the path
+    return the index-th part.
+    """
+
+    without_protocol = exp.SplitPart(
+        this=url,
+        delimiter=exp.Literal.string("://"),
+        part_index=exp.Literal.number(2)
+    )
+
+    part = exp.SplitPart(
+        this=without_protocol,
+        delimiter=exp.Literal.string("/"),
+        part_index=exp.Literal.number(index)
+    )
+
+    return part

warehouse/metrics_mesh/models/intermediate/blockchain_artifacts/int_contracts_by_project.sql

@@ -8,7 +8,7 @@ with contracts_in_ossd as (
     project_id,
     artifact_source as chain,
     artifact_name as contract_address
-  from metrics.int_artifacts_in_ossd_by_project
+  from metrics.int_artifacts_by_project_in_ossd
   where artifact_type = 'CONTRACT'
 ),
 

warehouse/metrics_mesh/models/intermediate/code_artifacts/int_repositories.sql

@@ -22,6 +22,6 @@ select
   repos.language,
   repos.created_at,
   repos.updated_at
-from metrics.int_artifacts_in_ossd_by_project as artifacts
+from metrics.int_artifacts_by_project_in_ossd as artifacts
 inner join metrics.stg_ossd__current_repositories as repos
   on artifacts.artifact_source_id = CAST(repos.id as STRING)

warehouse/metrics_mesh/models/intermediate/directory/int_all_artifacts.sql

@@ -22,7 +22,7 @@ with ossd_artifacts as (
     artifact_namespace,
     artifact_name,
     artifact_url
-  from metrics.int_artifacts_in_ossd_by_project
+  from metrics.int_artifacts_by_project_in_ossd
   where artifact_type not in ('DEPLOYER', 'CONTRACT')
 ),
 

warehouse/metrics_mesh/models/intermediate/directory/int_artifacts_by_project_in_op_atlas.sql

@@ -0,0 +1,114 @@
+MODEL (
+  name metrics.int_artifacts_by_project_in_op_atlas,
+  kind FULL,
+  dialect trino
+);
+
+with all_websites as (
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_namespace,
+    artifact_name,
+    artifact_url,
+    artifact_type
+  from metrics.stg_op_atlas_project_website as sites
+),
+
+all_farcaster as (
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_namespace,
+    artifact_name,
+    artifact_url,
+    artifact_type
+  from metrics.stg_op_atlas_project_farcaster as farcaster
+),
+
+all_twitter as (
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_namespace,
+    artifact_name,
+    artifact_url,
+    artifact_type
+  from metrics.stg_op_atlas_project_twitter as twitter 
+),
+
+all_repository as (
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_namespace,
+    artifact_name,
+    artifact_url,
+    artifact_type
+  from metrics.stg_op_atlas_project_repository
+),
+
+all_contracts as (
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_namespace,
+    artifact_name,
+    artifact_url,
+    artifact_type
+  from metrics.stg_op_atlas_project_contract
+),
+
+all_defillama as (
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_namespace,
+    artifact_name,
+    artifact_url,
+    artifact_type
+  from metrics.stg_op_atlas_project_defillama
+),
+
+all_artifacts as (
+  select * from all_websites
+  union all
+  select * from all_farcaster
+  union all
+  select * from all_twitter
+  union all
+  select * from all_repository
+  union all
+  select * from all_contracts
+  union all
+  select * from all_defillama
+),
+
+all_normalized_artifacts as (
+  select distinct
+    project_id,
+    LOWER(artifact_source_id) as artifact_source_id,
+    UPPER(artifact_source) as artifact_source,
+    LOWER(artifact_namespace) as artifact_namespace,
+    LOWER(artifact_name) as artifact_name,
+    LOWER(artifact_url) as artifact_url,
+    UPPER(artifact_type) as artifact_type
+  from all_artifacts
+)
+
+select
+  project_id,
+  @oso_id(artifact_source, artifact_source_id) as artifact_id,
+  artifact_source_id,
+  artifact_source,
+  artifact_namespace,
+  artifact_name,
+  artifact_url,
+  artifact_type
+from all_normalized_artifacts

warehouse/metrics_mesh/models/intermediate/directory/int_artifacts_in_ossd_by_project.sql → warehouse/metrics_mesh/models/intermediate/directory/int_artifacts_by_project_in_ossd.sql

@@ -1,5 +1,5 @@
 MODEL (
-  name metrics.int_artifacts_in_ossd_by_project,
+  name metrics.int_artifacts_by_project_in_ossd,
   kind FULL,
   dialect trino
 );

warehouse/metrics_mesh/models/intermediate/directory/int_deployers_by_project.sql

@@ -14,7 +14,7 @@ known_deployers as (
     project_id,
     artifact_source,
     artifact_name
-  from metrics.int_artifacts_in_ossd_by_project
+  from metrics.int_artifacts_by_project_in_ossd
   where artifact_type = 'DEPLOYER'
 ),
 

warehouse/metrics_mesh/models/intermediate/directory/int_projects.sql

@@ -4,14 +4,31 @@ MODEL (
   kind FULL
 );
 
-select
-  project_id,
-  project_source,
-  project_namespace,
-  project_name,
-  display_name,
-  description,
-  ARRAY_LENGTH(github) as github_artifact_count,
-  ARRAY_LENGTH(blockchain) as blockchain_artifact_count,
-  ARRAY_LENGTH(npm) as npm_artifact_count
-from metrics.stg_ossd__current_projects
+with ossd_projects as (
+  select
+    project_id,
+    project_source,
+    project_namespace,
+    project_name,
+    display_name,
+    description,
+    --ARRAY_LENGTH(github) as github_artifact_count,
+    --ARRAY_LENGTH(blockchain) as blockchain_artifact_count,
+    --ARRAY_LENGTH(npm) as npm_artifact_count
+  from metrics.stg_ossd__current_projects
+),
+
+op_atlas_projects as (
+  select
+    project_id,
+    project_source,
+    project_namespace,
+    project_name,
+    display_name,
+    description
+  from metrics.stg_op_atlas_project
+)
+
+select * from ossd_projects
+union all
+select * from op_atlas_projects

warehouse/metrics_mesh/models/staging/op_atlas/stg_op_atlas_project.sql

@@ -6,6 +6,7 @@ MODEL (
 
 select
   @oso_id('OP_ATLAS', id)::TEXT as project_id,
+  id::TEXT as project_source_id,
   'OP_ATLAS' as project_source,
   null::TEXT as project_namespace,
   id::TEXT as project_name,

warehouse/metrics_mesh/models/staging/op_atlas/stg_op_atlas_project_contract.sql

@@ -7,11 +7,17 @@ MODEL (
 select
   -- Translating op-atlas project_id to OSO project_id
   @oso_id('OP_ATLAS', project_id) as project_id,
-  created_at,
-  updated_at,
-  chain_id,
-  deployer_address,
-  contract_address,
-  deployment_hash,
-  verification_proof
+  id as artifact_source_id,
+  @chain_id_to_chain_name(chain_id) as artifact_source,
+  NULL::TEXT as artifact_namespace,
+  contract_address as artifact_name,
+  NULL::TEXT as artifact_url,
+  'CONTRACT' as artifact_type,
+  --created_at,
+  --updated_at,
+  --chain_id,
+  --deployer_address,
+  --contract_address,
+  --deployment_hash,
+  --verification_proof
 from @oso_source('bigquery.op_atlas.project_contract')

warehouse/metrics_mesh/models/staging/op_atlas/stg_op_atlas_project_defillama.sql

@@ -0,0 +1,18 @@
+MODEL (
+  name metrics.stg_op_atlas_project_defillama,
+  dialect trino,
+  kind FULL,
+);
+
+select
+    -- Translating op-atlas project_id to OSO project_id
+    @oso_id('OP_ATLAS', projects.id) as project_id,
+    defillama._dlt_id as artifact_source_id,
+    'DEFILLAMA' as artifact_source,
+    'DEFILLAMA' as artifact_namespace,
+    defillama.value as artifact_name,
+    concat('https://defillama.com/protocol/', defillama.value) as artifact_url,
+    'DEFILLAMA' as artifact_type
+from @oso_source('bigquery.op_atlas.project__defi_llama_slug') as defillama
+left join @oso_source('bigquery.op_atlas.project') as projects
+on defillama._dlt_parent_id = projects._dlt_id