Update to support merging EPCI data

datagouv · Apr 11, 2023 · b62c27d · b62c27d
1 parent 8f325eb
commit b62c27d
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 58 deletions.
diff --git a/bin/cli.js b/bin/cli.js
@@ -89,12 +89,14 @@ program
 
 program
   .command('merge-epci <workDir>')
+  .option('--from <bundleType>', 'Type d’archive source : etalab ou pci')
   .description('merge communes into EPCI')
-  .action(workDir => {
+  .action((workDir, {from}) => {
     if (!workDir) throw new Error('workDir is required')
     workDir = resolve(workDir)
+    if (!from || !['etalab', 'pci'].includes(from)) throw new Error('from est obligatoire et doit être choisi parmi etalab ou pci')
 
-    mergeEpciCmd(workDir).catch(boom)
+    mergeEpciCmd(workDir, from).catch(boom)
   })
 
 program

diff --git a/lib/commands/merge-epci.js b/lib/commands/merge-epci.js
@@ -4,9 +4,12 @@ import {join} from 'node:path'
 import {createRequire} from 'node:module'
 
 import {pathExists} from 'fs-extra'
+import bluebird from 'bluebird'
 import {ensureDirectoryExists} from '../util/fs.js'
 import {mergeGeoJSONFiles} from '../merge/index.js'
 import {getGeoJSONPath} from '../dist/simple.js'
+import {Tree, FORMATS} from '../dist/pci.js'
+import {WritableZipFile} from '../util/zip.js'
 
 const require = createRequire(import.meta.url)
 
@@ -57,46 +60,86 @@ const GEOJSON_RAW_LAYERS = [
   'label',
 ]
 
-async function handler(workDir) {
-  await Promise.all(Object.entries(epciCommunesMembers).map(async ([epci, communes]) => {
-    console.log('  merging epci %s', epci)
-    console.time('  merged epci ' + epci)
-
-    await Promise.all(GEOJSON_LAYERS.map(async layer => {
-      const files = communes.map(commune => {
-        const departement = `${commune.startsWith('97') ? commune.slice(0, 3) : commune.slice(0, 2)}`
-        return join(getGeoJSONPath(workDir), 'communes', departement, `${commune}`, `cadastre-${commune}-${layer}.json.gz`)
-      })
-      const filteredFiles = await asyncFilter(files, pathExists)
-      const targetDir = join(getGeoJSONPath(workDir), 'epci', epci)
-      await ensureDirectoryExists(targetDir)
-      const targetPath = join(getGeoJSONPath(workDir), 'epci', epci, `pci-epci-${epci}-${layer}.json.gz`)
-      await mergeGeoJSONFiles({srcFiles: filteredFiles, destPath: targetPath})
-      console.log(`    ${epci} | merged ${layer}`)
+async function handler(workDir, from) {
+  if (from === 'etalab') {
+    await Promise.all(Object.entries(epciCommunesMembers).map(async ([epci, communes]) => {
+      console.log('  merging epci %s', epci)
+      console.time('  merged epci ' + epci)
+
+      await Promise.all(GEOJSON_LAYERS.map(async layer => {
+        const files = communes.map(commune => {
+          const departement = `${commune.startsWith('97') ? commune.slice(0, 3) : commune.slice(0, 2)}`
+          return join(getGeoJSONPath(workDir), 'communes', departement, `${commune}`, `cadastre-${commune}-${layer}.json.gz`)
+        })
+        const filteredFiles = await asyncFilter(files, pathExists)
+        const targetDir = join(getGeoJSONPath(workDir), 'epci', epci)
+        await ensureDirectoryExists(targetDir)
+        const targetPath = join(getGeoJSONPath(workDir), 'epci', epci, `epci-${epci}-${layer}.json.gz`)
+        await mergeGeoJSONFiles({srcFiles: filteredFiles, destPath: targetPath})
+        console.log(`    ${epci} | merged ${layer}`)
+      }))
+
+      console.timeEnd('  merged epci ' + epci)
     }))
 
-    console.timeEnd('  merged epci ' + epci)
-  }))
-
-  await Promise.all(Object.entries(epciCommunesMembers).map(async ([epci, communes]) => {
-    console.log('  [raw] merging epci %s', epci)
-    console.time('  [raw] merged epci ' + epci)
-
-    await Promise.all(GEOJSON_RAW_LAYERS.map(async layer => {
-      const files = communes.map(commune => {
-        const departement = `${commune.startsWith('97') ? commune.slice(0, 3) : commune.slice(0, 2)}`
-        return join(getGeoJSONPath(workDir), 'communes', departement, `${commune}`, 'raw', `cadastre-${commune}-${layer}.json.gz`)
-      })
-      const filteredFiles = await asyncFilter(files, pathExists)
-      const targetDir = join(getGeoJSONPath(workDir), 'epci', epci, 'raw')
-      await ensureDirectoryExists(targetDir)
-      const targetPath = join(getGeoJSONPath(workDir), 'epci', epci, 'raw', `pci-epci-${epci}-${layer}.json.gz`)
-      await mergeGeoJSONFiles({srcFiles: filteredFiles, destPath: targetPath})
-      console.log(`    ${epci} | merged ${layer}`)
+    await Promise.all(Object.entries(epciCommunesMembers).map(async ([epci, communes]) => {
+      console.log('  [raw] merging epci %s', epci)
+      console.time('  [raw] merged epci ' + epci)
+
+      await Promise.all(GEOJSON_RAW_LAYERS.map(async layer => {
+        const files = communes.map(commune => {
+          const departement = `${commune.startsWith('97') ? commune.slice(0, 3) : commune.slice(0, 2)}`
+          return join(getGeoJSONPath(workDir), 'communes', departement, `${commune}`, 'raw', `cadastre-${commune}-${layer}.json.gz`)
+        })
+        const filteredFiles = await asyncFilter(files, pathExists)
+        const targetDir = join(getGeoJSONPath(workDir), 'epci', epci, 'raw')
+        await ensureDirectoryExists(targetDir)
+        const targetPath = join(getGeoJSONPath(workDir), 'epci', epci, 'raw', `epci-${epci}-${layer}.json.gz`)
+        await mergeGeoJSONFiles({srcFiles: filteredFiles, destPath: targetPath})
+        console.log(`    ${epci} | merged ${layer}`)
+      }))
+
+      console.timeEnd('  [raw] merged epci ' + epci)
     }))
-
-    console.timeEnd('  [raw] merged epci ' + epci)
-  }))
+  }
+
+  if (from === 'pci') {
+    await bluebird.map(Object.entries(epciCommunesMembers), async ([epci, communes]) => {
+      await Promise.all(FORMATS.filter(formatInfos => formatInfos.name !== 'tiff').map(formatInfos => formatInfos.name).map(async format => {
+        console.log('  merging epci %s format %s', epci, format)
+        console.time(`  merging epci ${epci} files ${format}`)
+
+        const tree = new Tree(workDir, 'dgfip-pci-vecteur', format)
+        const filteredCommunes = await asyncFilter(communes, async commune => {
+          const departement = `${commune.startsWith('97') ? commune.slice(0, 3) : commune.slice(0, 2)}`
+          return pathExists(`${tree.getFeuillesBasePath()}/${departement}/${commune}`)
+        })
+        const feuillesCommunes = await Promise.all(filteredCommunes.map(async codeCommune => ({
+          code: codeCommune,
+          feuilles: await tree.listFeuillesByCommune(codeCommune),
+        })))
+        // eslint-disable-next-line unicorn/no-array-reduce
+        const filesToZip = feuillesCommunes.reduce((acc, feuillesCommune) => {
+          acc = [...acc, ...feuillesCommune.feuilles.map(feuille => {
+            const dest = `${tree.getFeuillePathInArchive(feuille)}`
+            return [tree.getFeuillePath(feuille), dest]
+          })]
+          return acc
+        }, [])
+        const targetDir = `${tree.treeBasePath}/epci/`
+
+        const destArchive = new WritableZipFile(`${targetDir}/cadastre-${epci}-${format}.zip`)
+        await bluebird.map(filesToZip, async fileInfos => {
+          const [file, dest] = fileInfos
+          destArchive.addFile(file, dest, {compress: false})
+        }, {concurrency: 4})
+        await destArchive.end()
+        console.log('end zip')
+
+        console.timeEnd(`  merging epci ${epci} files ${format}`)
+      }))
+    }, {concurrency: 4})
+  }
 
   console.log('Finished!')
 

diff --git a/yarn.lock b/yarn.lock
@@ -3688,13 +3688,6 @@ equal-length@^1.0.0:
   resolved "https://registry.yarnpkg.com/equal-length/-/equal-length-1.0.1.tgz#21ca112d48ab24b4e1e7ffc0e5339d31fdfc274c"
   integrity sha512-TK2m7MvWPt/v3dan0BCNp99pytIE5UGrUj7F0KZirNX8xz8fDFUAZfgm8uB5FuQq9u0sMeDocYBfEhsd1nwGoA==
 
-errno@~0.1.7:
-  version "0.1.8"
-  resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.8.tgz#8bb3e9c7d463be4976ff888f76b4809ebc2e811f"
-  integrity sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==
-  dependencies:
-    prr "~1.0.1"
-
 error-ex@^1.3.1:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf"
@@ -6273,11 +6266,6 @@ proto-props@^2.0.0:
   resolved "https://registry.yarnpkg.com/proto-props/-/proto-props-2.0.0.tgz#8ac6e6dec658545815c623a3bc81580deda9a181"
   integrity sha512-2yma2tog9VaRZY2mn3Wq51uiSW4NcPYT1cQdBagwyrznrilKSZwIZ0UG3ZPL/mx+axEns0hE35T5ufOYZXEnBQ==
 
-prr@~1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"
-  integrity sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==
-
 pseudomap@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
@@ -7365,13 +7353,6 @@ wordwrap@~0.0.2:
   resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107"
   integrity sha512-1tMA907+V4QmxV7dbRvb4/8MaRALK6q9Abid3ndMYnbyo8piisCmeONVqVSXqQA3KaP4SLt5b7ud6E2sqP8TFw==
 
-worker-farm@^1.7.0:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8"
-  integrity sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==
-  dependencies:
-    errno "~0.1.7"
-
 wrap-ansi@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"