feat: allow copying maps to device memory from external providers (eg…

… zip, remote) from pause menu

feat: display broken worlds warning
fix: fs didn't reset on world unload
feat: allow full warps support (eg Greenfield maps)! with a way to set your own

package.json

@@ -49,7 +49,7 @@
     "esbuild": "^0.19.3",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "express": "^4.18.2",
-    "flying-squid": "npm:@zardoy/flying-squid@^0.0.10",
+    "flying-squid": "npm:@zardoy/flying-squid@^0.0.12",
     "fs-extra": "^11.1.1",
     "iconify-icon": "^1.0.8",
     "jszip": "^3.10.1",

src/browserfs.ts

@@ -28,6 +28,7 @@ browserfs.configure({
 })
 
 export const forceCachedDataPaths = {}
+export const forceRedirectPaths = {}
 
 //@ts-expect-error
 fs.promises = new Proxy(Object.fromEntries(['readFile', 'writeFile', 'stat', 'mkdir', 'rmdir', 'unlink', 'rename', /* 'copyFile',  */'readdir'].map(key => [key, promisify(fs[key])])), {
@@ -36,14 +37,20 @@ fs.promises = new Proxy(Object.fromEntries(['readFile', 'writeFile', 'stat', 'mk
     return (...args) => {
       // browser fs bug: if path doesn't start with / dirname will return . which would cause infinite loop, so we need to normalize paths
       if (typeof args[0] === 'string' && !args[0].startsWith('/')) args[0] = '/' + args[0]
+      const toRemap = Object.entries(forceRedirectPaths).find(([from]) => args[0].startsWith(from))
+      if (toRemap) {
+        args[0] = args[0].replace(toRemap[0], toRemap[1])
+      }
       // Write methods
       // todo issue one-time warning (in chat I guess)
-      if (fsState.isReadonly) {
+      const readonly = fsState.isReadonly && !(args[0].startsWith('/data') && !fsState.inMemorySave) // allow copying worlds from external providers such as zip
+      if (readonly) {
         if (oneOf(p, 'readFile', 'writeFile') && forceCachedDataPaths[args[0]]) {
           if (p === 'readFile') {
             return Promise.resolve(forceCachedDataPaths[args[0]])
           } else if (p === 'writeFile') {
             forceCachedDataPaths[args[0]] = args[1]
+            console.debug('Skipped writing to readonly fs', args[0])
             return Promise.resolve()
           }
         }
@@ -322,7 +329,19 @@ export const possiblyCleanHandle = (callback = () => { }) => {
   }
 }
 
-export const copyFilesAsyncWithProgress = async (pathSrc: string, pathDest: string) => {
+export const copyFilesAsyncWithProgress = async (pathSrc: string, pathDest: string, throwRootNotExist = true) => {
+  const stat = await existsViaStats(pathSrc)
+  if (!stat) {
+    if (throwRootNotExist) throw new Error(`Cannot copy. Source directory ${pathSrc} does not exist`)
+    console.debug('source directory does not exist', pathSrc)
+    return
+  }
+  if (!stat.isDirectory()) {
+    await fs.promises.writeFile(pathDest, await fs.promises.readFile(pathSrc))
+    console.debug('copied single file', pathSrc, pathDest)
+    return
+  }
+
   try {
     setLoadingScreenStatus('Copying files')
     let filesCount = 0
@@ -339,21 +358,35 @@ export const copyFilesAsyncWithProgress = async (pathSrc: string, pathDest: stri
         }
       }))
     }
+    console.debug('Counting files', pathSrc)
     await countFiles(pathSrc)
+    console.debug('counted', filesCount)
     let copied = 0
     await copyFilesAsync(pathSrc, pathDest, (name) => {
       copied++
-      setLoadingScreenStatus(`Copying files (${copied}/${filesCount}) ${name}...`)
+      setLoadingScreenStatus(`Copying files (${copied}/${filesCount}): ${name}`)
     })
   } finally {
     setLoadingScreenStatus(undefined)
   }
 }
 
+export const existsViaStats = async (path: string) => {
+  try {
+    return await fs.promises.stat(path)
+  } catch (e) {
+    return false
+  }
+}
+
 export const copyFilesAsync = async (pathSrc: string, pathDest: string, fileCopied?: (name) => void) => {
   // query: can't use fs.copy! use fs.promises.writeFile and readFile
   const files = await fs.promises.readdir(pathSrc)
 
+  if (!await existsViaStats(pathDest)) {
+    await fs.promises.mkdir(pathDest, { recursive: true })
+  }
+
   // Use Promise.all to parallelize file/directory copying
   await Promise.all(files.map(async (file) => {
     const curPathSrc = join(pathSrc, file)
@@ -365,8 +398,14 @@ export const copyFilesAsync = async (pathSrc: string, pathDest: string, fileCopi
       await copyFilesAsync(curPathSrc, curPathDest, fileCopied)
     } else {
       // Copy file
-      await fs.promises.writeFile(curPathDest, await fs.promises.readFile(curPathSrc))
-      fileCopied?.(file)
+      try {
+        await fs.promises.writeFile(curPathDest, await fs.promises.readFile(curPathSrc))
+        console.debug('copied file', curPathSrc, curPathDest)
+      } catch (err) {
+        console.error('Error copying file', curPathSrc, curPathDest, err)
+        throw err
+      }
+      fileCopied?.(curPathDest)
     }
   }))
 }

src/index.ts

@@ -355,13 +355,16 @@ async function connect (connectOptions: {
       //@ts-expect-error
       window.bot = bot = undefined
     }
+    resetStateAfterDisconnect()
+    cleanFs()
+    removeAllListeners()
+  }
+  const cleanFs = () => {
     if (singleplayer && !fsState.inMemorySave) {
       possiblyCleanHandle(() => {
         // todo: this is not enough, we need to wait for all async operations to finish
       })
     }
-    resetStateAfterDisconnect()
-    removeAllListeners()
   }
   const onPossibleErrorDisconnect = () => {
     // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison

src/loadSave.ts

@@ -1,11 +1,12 @@
 import fs from 'fs'
+import path from 'path'
 import { supportedVersions } from 'flying-squid/dist/lib/version'
 import * as nbt from 'prismarine-nbt'
 import { proxy } from 'valtio'
 import { gzip } from 'node-gzip'
 import { options } from './optionsStorage'
 import { nameToMcOfflineUUID, disconnect } from './flyingSquidUtils'
-import { forceCachedDataPaths } from './browserfs'
+import { forceCachedDataPaths, forceRedirectPaths, mkdirRecursive } from './browserfs'
 import { isMajorVersionGreater } from './utils'
 
 import { activeModalStacks, insertActiveModalStack, miscUiState } from './globalState'
@@ -43,6 +44,14 @@ export const readLevelDat = async (path) => {
 }
 
 export const loadSave = async (root = '/world') => {
+  // todo test
+  if (miscUiState.gameLoaded) {
+    await disconnect()
+    await new Promise(resolve => {
+      setTimeout(resolve)
+    })
+  }
+
   const disablePrompts = options.disableLoadPrompts
 
   // todo do it in singleplayer as well
@@ -51,6 +60,10 @@ export const loadSave = async (root = '/world') => {
     // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
     delete forceCachedDataPaths[key]
   }
+  for (const key in forceRedirectPaths) {
+    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+    delete forceRedirectPaths[key]
+  }
   // todo check jsHeapSizeLimit
 
   const warnings: string[] = []
@@ -105,6 +118,7 @@ export const loadSave = async (root = '/world') => {
       if (fsState.isReadonly) {
         forceCachedDataPaths[playerDatPath] = playerDat
       } else {
+        await mkdirRecursive(path.dirname(playerDatPath))
         await fs.promises.writeFile(playerDatPath, playerDat)
       }
     }
@@ -136,10 +150,12 @@ export const loadSave = async (root = '/world') => {
     alert('Note: the world is saved only on /save or disconnect! Ensure you have backup!')
   }
 
-  // todo fix these
-  if (miscUiState.gameLoaded) {
-    await disconnect()
+  // improve compatibility with community saves
+  const rootRemapFiles = ['Warp files']
+  for (const rootRemapFile of rootRemapFiles) {
+    forceRedirectPaths[path.join(root, rootRemapFile)] = path.join(root, '..', rootRemapFile)
   }
+
   // todo reimplement
   if (activeModalStacks['main-menu']) {
     insertActiveModalStack('main-menu')

src/menus/pause_screen.js

@@ -1,15 +1,18 @@
 //@ts-check
+const { join } = require('path')
 const { LitElement, html, css } = require('lit')
 const { subscribe } = require('valtio')
 const { subscribeKey } = require('valtio/utils')
+const { usedServerPathsV1 } = require('flying-squid/dist/lib/modules/world')
 const { hideCurrentModal, showModal, miscUiState, notification, openOptionsMenu } = require('../globalState')
 const { fsState } = require('../loadSave')
-const { openGithub } = require('../utils')
+const { openGithub, setLoadingScreenStatus } = require('../utils')
 const { disconnect } = require('../flyingSquidUtils')
 const { closeWan, openToWanAndCopyJoinLink, getJoinLink } = require('../localServerMultiplayer')
-const { uniqueFileNameFromWorldName, copyFilesAsyncWithProgress } = require('../browserfs')
+const { uniqueFileNameFromWorldName, copyFilesAsyncWithProgress, existsViaStats, mkdirRecursive } = require('../browserfs')
 const { showOptionsModal } = require('../react/SelectOption')
 const { openURL } = require('./components/common')
+const fs = require('fs').promises
 
 class PauseScreen extends LitElement {
   static get styles () {
@@ -69,10 +72,23 @@ class PauseScreen extends LitElement {
     }
     const action = await showOptionsModal('World actions...', ['Save to browser memory'])
     if (action === 'Save to browser memory') {
-      //@ts-expect-error
-      const { worldFolder } = localServer.options
-      const savePath = await uniqueFileNameFromWorldName(worldFolder.split('/').pop(), `/data/worlds`)
-      await copyFilesAsyncWithProgress(worldFolder, savePath)
+      setLoadingScreenStatus('Saving world')
+      try {
+        //@ts-expect-error
+        const { worldFolder } = localServer.options
+        const saveRootPath = await uniqueFileNameFromWorldName(worldFolder.split('/').pop(), `/data/worlds`)
+        await mkdirRecursive(saveRootPath)
+        for (const copyPath of [...usedServerPathsV1, 'icon.png']) {
+          const srcPath = join(worldFolder, copyPath)
+          const savePath = join(saveRootPath, copyPath)
+          // eslint-disable-next-line no-await-in-loop
+          await copyFilesAsyncWithProgress(srcPath, savePath, false)
+        }
+      } catch (err) {
+        void showOptionsModal(`Error while saving the world: ${err.message}`, [])
+      } finally {
+        setLoadingScreenStatus(undefined)
+      }
     }
   }
 
@@ -83,7 +99,7 @@ class PauseScreen extends LitElement {
     return html`
       <div class="bg"></div>
       <!-- todo uncomment when browserfs is fixed -->
-      <!--<pmui-button style="position:fixed;left: 5px;top: 5px;" pmui-icon="pixelarticons:folder" pmui-width="20px" pmui-label="" @pmui-click=${async () => this.openWorldActions()}></pmui-button>-->
+      <pmui-button style="position:fixed;left: 5px;top: 5px;" pmui-icon="pixelarticons:folder" pmui-width="20px" pmui-label="" @pmui-click=${async () => this.openWorldActions()}></pmui-button>
 
       <p class="title">Game Menu</p>
 

src/react/Singleplayer.tsx

@@ -65,12 +65,15 @@ interface Props {
   disabledProviders?: string[]
   isReadonly?: boolean
   error?: string
+  warning?: string
+  warningAction?: () => void
+  warningActionLabel?: string
 
   onWorldAction (action: 'load' | 'export' | 'delete' | 'edit', worldName: string): void
   onGeneralAction (action: 'cancel' | 'create'): void
 }
 
-export default ({ worldData, onGeneralAction, onWorldAction, activeProvider, setActiveProvider, providerActions, providers, disabledProviders, error, isReadonly }: Props) => {
+export default ({ worldData, onGeneralAction, onWorldAction, activeProvider, setActiveProvider, providerActions, providers, disabledProviders, error, isReadonly, warning, warningAction, warningActionLabel }: Props) => {
   const containerRef = useRef<any>()
   const firstButton = useRef<HTMLButtonElement>(null!)
 
@@ -129,7 +132,17 @@ export default ({ worldData, onGeneralAction, onWorldAction, activeProvider, set
               : <div style={{
                 fontSize: 10,
                 color: error ? 'red' : 'lightgray',
-              }}>{error || 'Loading (#dev check console if loading too long)...'}</div>
+              }}>{error || 'Loading (check #dev console if loading too long)...'}</div>
+          }
+          {
+            warning && <div style={{
+              fontSize: 8,
+              color: '#ffa500ba',
+              marginTop: 5,
+              textAlign: 'center',
+            }}>
+              {warning} {warningAction && <a onClick={warningAction}>{warningActionLabel}</a>}
+            </div>
           }
         </div>
       </div>

src/react/SingleplayerProvider.tsx

@@ -16,6 +16,7 @@ import GoogleButton from './GoogleButton'
 
 const worldsProxy = proxy({
   value: null as null | WorldProps[],
+  brokenWorlds: [] as string[],
   selectedProvider: 'local' as 'local' | 'google',
   error: '',
 })
@@ -40,8 +41,10 @@ const providersEnableFeatures = {
 
 export const readWorlds = (abortController: AbortController) => {
   if (abortController.signal.aborted) return
-  worldsProxy.error = '';
+  worldsProxy.error = ''
+  worldsProxy.brokenWorlds = [];
   (async () => {
+    const brokenWorlds = [] as string[]
     try {
       const loggedIn = !!googleProviderData.accessToken
       worldsProxy.value = null
@@ -72,9 +75,10 @@ export const readWorlds = (abortController: AbortController) => {
           detail: `${levelDat.Version?.Name ?? 'unknown version'}, ${folder}`,
           size,
         } satisfies WorldProps
-      }))).filter(x => {
+      }))).filter((x, i) => {
         if (x.status === 'rejected') {
           console.warn(x.reason)
+          brokenWorlds.push(worlds[i])
           return false
         }
         return true
@@ -87,6 +91,7 @@ export const readWorlds = (abortController: AbortController) => {
       worldsProxy.value = null
       worldsProxy.error = err.message
     }
+    worldsProxy.brokenWorlds = brokenWorlds
   })().catch((err) => {
     // todo it still doesn't work for some reason!
     worldsProxy.error = err.message
@@ -128,7 +133,7 @@ export const loadGoogleDriveApi = async () => {
 
 const Inner = () => {
   const worlds = useSnapshot(worldsProxy).value as WorldProps[] | null
-  const { selectedProvider, error } = useSnapshot(worldsProxy)
+  const { selectedProvider, error, brokenWorlds } = useSnapshot(worldsProxy)
   const readWorldsAbortController = useRef(new AbortController())
 
   useEffect(() => {
@@ -260,6 +265,16 @@ const Inner = () => {
     setActiveProvider={(provider) => {
       worldsProxy.selectedProvider = provider as any
     }}
+    warning={brokenWorlds.length ? `Some worlds are broken: ${brokenWorlds.join(', ')}` : undefined}
+    warningAction={async () => {
+      for (const brokenWorld of worldsProxy.brokenWorlds) {
+        setLoadingScreenStatus(`Removing broken world ${brokenWorld}`)
+        // eslint-disable-next-line no-await-in-loop
+        await removeFileRecursiveAsync(`${getWorldsPath()}/${brokenWorld}`)
+      }
+      setLoadingScreenStatus(undefined)
+    }}
+    warningActionLabel='Remove broken worlds'
     onWorldAction={async (action, worldName) => {
       const worldPath = `${getWorldsPath()}/${worldName}`
       const openInGoogleDrive = () => {