Merge pull request #1256 from nickcastel50/optimized_model_loading

Optimized Model Loading
bldrs-ai · Sep 13, 2024 · 01b8ff4 · 01b8ff4
2 parents 9c58b7e + 800e1f2
commit 01b8ff4
Show file tree

Hide file tree

Showing 22 changed files with 1,335 additions and 243 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "bldrs",
-  "version": "1.0.1103",
+  "version": "1.0.1101",
   "main": "src/index.jsx",
   "license": "AGPL-3.0",
   "homepage": "https://github.com/bldrs-ai/Share",
@@ -12,7 +12,7 @@
     "build-conway": "yarn clean && yarn build-share-conway && yarn build-cosmos",
     "build-webifc": "yarn clean && yarn build-share-webifc && yarn build-cosmos",
     "build-cosmos": "shx mkdir -p docs ; shx rm -rf docs/cosmos; cosmos-export --config .cosmos.config.json && shx mv cosmos-export docs/cosmos",
-    "build-share": "yarn update-version && node tools/esbuild/build.js && shx mkdir -p docs/static/js && shx cp src/OPFS/OPFS.worker.js docs/",
+    "build-share": "yarn update-version && node tools/esbuild/build.js && shx mkdir -p docs/static/js && shx cp src/OPFS/OPFS.worker.js docs/ && shx cp src/net/github/Cache.js docs/",
     "build-share-conway": "run-script-os",
     "build-share-conway:win32": "set USE_WEBIFC_SHIM=true&& yarn build-share",
     "build-share-conway:linux:darwin": "USE_WEBIFC_SHIM=true yarn build-share",

diff --git a/src/Components/Open/SaveModelControl.jsx b/src/Components/Open/SaveModelControl.jsx
@@ -232,63 +232,67 @@ function SaveModelDialog({isDialogDisplayed, setIsDialogDisplayed, navigate, org
         justifyContent='center'
         alignItems='center'
       >
-        {!isAuthenticated ?
-
-         <PleaseLogin/> :
-
-         <Stack>
-           <Typography variant='overline' sx={{marginBottom: '6px'}}>Projects</Typography>
-           <Selector label='Organization' list={orgNamesArrWithAt} selected={selectedOrgName} setSelected={selectOrg}
-           data-testid='saveOrganization'
-           />
-           <Selector
-             label='Repository'
-             list={repoNamesArr}
-             selected={selectedRepoName}
-             setSelected={selectRepo}
-             data-testid='saveRepository'
-           />
-           <SelectorSeparator
-             label={(currentPath === '') ? 'Folder' :
-                    `Folder: ${currentPath}`}
-             list={foldersArr}
-             selected={selectedFolderName}
-             setSelected={selectFolder}
-             data-testid='saveFolder'
-           />
-           {requestCreateFolder && (
-             <div style={{display: 'flex', alignItems: 'center', marginBottom: '.5em'}}>
-               <TextField
-                 label='Enter folder name'
-                 variant='outlined'
-                 size='small'
-                 onChange={(e) => setCreateFolderName(e.target.value)}
-                 data-testid='CreateFolderId'
-                 sx={{flexGrow: 1}}
-                 onKeyDown={(e) => {
-                   // Stops the event from propagating up to parent elements
-                   e.stopPropagation()
-                 }}
-               />
-               <IconButton
-                 onClick={() => setRequestCreateFolder(false)}
-                 size='small'
-               >
-                 <ClearIcon className='icon-share'/>
-               </IconButton>
-             </div>
-           )}
-           <TextField
-             label='Enter file name'
-             variant='outlined'
-             size='small'
-             onChange={(e) => setSelectedFileName(e.target.value)}
-             onKeyDown={(e) => e.stopPropagation()}
-             sx={{marginBottom: '.5em'}}
-             data-testid='CreateFileId'
-           />
-         </Stack>
-        }
+        {!isAuthenticated ? (
+          <PleaseLogin/>
+        ) : (
+          file instanceof File && (
+            <Stack>
+              <Typography variant='overline' sx={{marginBottom: '6px'}}>Projects</Typography>
+              <Selector
+                label='Organization'
+                list={orgNamesArrWithAt}
+                selected={selectedOrgName}
+                setSelected={selectOrg}
+                data-testid='saveOrganization'
+              />
+              <Selector
+                label='Repository'
+                list={repoNamesArr}
+                selected={selectedRepoName}
+                setSelected={selectRepo}
+                data-testid='saveRepository'
+              />
+              <SelectorSeparator
+                label={currentPath === '' ? 'Folder' : `Folder: ${currentPath}`}
+                list={foldersArr}
+                selected={selectedFolderName}
+                setSelected={selectFolder}
+                data-testid='saveFolder'
+              />
+              {requestCreateFolder && (
+                <div style={{display: 'flex', alignItems: 'center', marginBottom: '.5em'}}>
+                  <TextField
+                    label='Enter folder name'
+                    variant='outlined'
+                    size='small'
+                    onChange={(e) => setCreateFolderName(e.target.value)}
+                    data-testid='CreateFolderId'
+                    sx={{flexGrow: 1}}
+                    onKeyDown={(e) => {
+                      // Stops the event from propagating up to parent elements
+                      e.stopPropagation()
+                    }}
+                  />
+                  <IconButton
+                    onClick={() => setRequestCreateFolder(false)}
+                    size='small'
+                  >
+                    <ClearIcon className='icon-share'/>
+                  </IconButton>
+                </div>
+              )}
+              <TextField
+                label='Enter file name'
+                variant='outlined'
+                size='small'
+                onChange={(e) => setSelectedFileName(e.target.value)}
+                onKeyDown={(e) => e.stopPropagation()}
+                sx={{marginBottom: '.5em'}}
+                data-testid='CreateFileId'
+              />
+            </Stack>
+          )
+        )}
       </Stack>
     </Dialog>
   )

diff --git a/src/Components/Open/SaveModelControl.test.jsx b/src/Components/Open/SaveModelControl.test.jsx
@@ -33,7 +33,7 @@ describe('SaveModelControl', () => {
     expect(loginText).toBeInTheDocument()
   })
 
-  it('Renders file selector if the user is logged in', async () => {
+   it('Renders file selector if the user is logged in', async () => {
     mockedUseAuth0.mockReturnValue(mockedUserLoggedIn)
     const {getByTestId} = render(<SaveModelControlFixture/>)
     const saveControlButton = getByTestId('control-button-save')

diff --git a/src/Components/Versions/VersionsPanel.fixture.js b/src/Components/Versions/VersionsPanel.fixture.js
@@ -1,4 +1,4 @@
-const RAW_GIT_PROXY_URL = process.env.RAW_GIT_PROXY_URL
+const RAW_GIT_PROXY_URL_NEW = process.env.RAW_GIT_PROXY_URL_NEW
 
 
 export const MOCK_MODEL_PATH_GIT = {
@@ -7,5 +7,5 @@ export const MOCK_MODEL_PATH_GIT = {
   branch: 'main',
   filepath: '/ZGRAGGEN.ifc',
   eltPath: '',
-  gitpath: `${RAW_GIT_PROXY_URL}/user2/Schneestock-Public/main/ZGRAGGEN.ifc`,
+  gitpath: `${RAW_GIT_PROXY_URL_NEW}/user2/Schneestock-Public/main/ZGRAGGEN.ifc`,
 }
diff --git a/src/Containers/CadView.jsx b/src/Containers/CadView.jsx
@@ -12,11 +12,10 @@ import ElementGroup from '../Components/ElementGroup'
 import HelpControl from '../Components/Help/HelpControl'
 import {useIsMobile} from '../Components/Hooks'
 import LoadingBackdrop from '../Components/LoadingBackdrop'
-import {getModelFromOPFS, downloadToOPFS} from '../OPFS/utils'
+import {getModelFromOPFS, downloadToOPFS, downloadModel} from '../OPFS/utils'
 import usePlaceMark from '../hooks/usePlaceMark'
 import * as Analytics from '../privacy/analytics'
 import useStore from '../store/useStore'
-import {getLatestCommitHash} from '../net/github/Commits'
 // TODO(pablo): use ^^ instead of this
 import {parseGitHubPath} from '../utils/location'
 import {getParentPathIdsForElement, setupLookupAndParentLinks} from '../utils/TreeUtils'
@@ -32,7 +31,7 @@ import ControlsGroupAndDrawer from './ControlsGroupAndDrawer'
 import OperationsGroupAndDrawer from './OperationsGroupAndDrawer'
 import ViewerContainer from './ViewerContainer'
 import {elementSelection} from './selection'
-import {getFinalUrl} from './urls'
+import {getFinalDownloadData} from './urls'
 import {initViewer} from './viewer'
 
 
@@ -252,8 +251,15 @@ export default function CadView({
 
     // NB: for LFS targets, this will now be media.githubusercontent.com, so
     // don't use for further API endpoint construction.
-    const ifcURL = (uploadedFile || filepath.indexOf('/') === 0) ?
-                   filepath : await getFinalUrl(filepath, accessToken)
+    let ifcURL; let shaHash
+
+    if (uploadedFile || filepath.indexOf('/') === 0) {
+        ifcURL = filepath
+        shaHash = ''
+    } else {
+        [ifcURL, shaHash] = await getFinalDownloadData(filepath, accessToken, isOpfsAvailable)
+    }
+
 
     const isCamHashSet = onHash(location, viewer.IFC.context.ifcCamera.cameraControls)
 
@@ -332,29 +338,21 @@ export default function CadView({
     } else {
       // TODO(pablo): probably already available in this scope, or use
       // parseGitHubRepositoryURL instead.
+      // eslint-disable-next-line no-unused-vars
       const {isPublic, owner, repo, branch, filePath} = parseGitHubPath(new URL(gitpath).pathname)
-      const commitHash = isPublic ?
-            await getLatestCommitHash(owner, repo, filePath, '', branch) :
-            await getLatestCommitHash(owner, repo, filePath, accessToken, branch)
-
-      if (commitHash === null) {
-        // downloadToOpfs below will error out as well.
-        debug().error(
-          `Error obtaining commit hash for: ` +
-            `owner:${owner}, repo:${repo}, filePath:${filePath}, branch:${branch} ` +
-            `accessToken (present?):${accessToken ? 'true' : 'false'}`)
-      }
 
-      const file = await downloadToOPFS(
+      const file = await downloadModel(
         navigate,
         appPrefix,
         handleBeforeUnload,
         ifcURL,
+        shaHash,
         filePath,
-        commitHash,
+        accessToken,
         owner,
         repo,
         branch,
+        setOpfsFile,
         (progressEvent) => {
           if (Number.isFinite(progressEvent.receivedLength)) {
             const loadedBytes = progressEvent.receivedLength
@@ -365,21 +363,17 @@ export default function CadView({
           }
         })
 
-      if (file instanceof File) {
-        setOpfsFile(file)
-      } else {
-        debug().error('Retrieved object is not of type File.')
+        if (file) {
+        loadedModel = await viewer.loadIfcFile(
+          file,
+          !isCamHashSet,
+          (error) => {
+            debug().log('CadView#loadIfc$onError: ', error)
+            // TODO(pablo): error modal.
+            setIsModelLoading(false)
+            setErrorPath(filePath)
+          }, customViewSettings)
       }
-
-      loadedModel = await viewer.loadIfcFile(
-        file,
-        !isCamHashSet,
-        (error) => {
-          debug().log('CadView#loadIfc$onError: ', error)
-          // TODO(pablo): error modal.
-          setIsModelLoading(false)
-          setErrorPath(filePath)
-        }, customViewSettings)
     }
 
     if (loadedModel) {

diff --git a/src/Containers/CadView.test.jsx b/src/Containers/CadView.test.jsx
@@ -49,7 +49,19 @@ jest.mock('../OPFS/utils', () => {
   return {
     ...actualUtils, // Preserve other exports from the module
     downloadToOPFS: jest.fn().mockImplementation(() => {
-      // Read the file content from disk (consider using async read in real use-cases)
+      // Read the file content from disk
+      const fileContent = fs.readFileSync(path.join(__dirname, './index.ifc'), 'utf8')
+
+      const uint8Array = new Uint8Array(fileContent)
+      const blob = new Blob([uint8Array])
+
+      // The lastModified property is optional, and can be omitted or set to Date.now() if needed
+      const file = new FileMock([blob], 'index.ifc', {type: 'text/plain', lastModified: Date.now()})
+      // Return the mocked File in a promise if it's an async function
+      return Promise.resolve(file)
+    }),
+    downloadModel: jest.fn().mockImplementation(() => {
+      // Read the file content from disk
       const fileContent = fs.readFileSync(path.join(__dirname, './index.ifc'), 'utf8')
 
       const uint8Array = new Uint8Array(fileContent)

diff --git a/src/Containers/urls.js b/src/Containers/urls.js
@@ -1,4 +1,4 @@
-import {getDownloadUrl} from '../net/github/Files'
+import {getDownloadUrl, getPathContents} from '../net/github/Files'
 import {parseGitHubRepositoryUrl} from '../net/github/utils'
 
 
@@ -12,7 +12,7 @@ export async function getFinalUrl(url, accessToken) {
   switch (u.host.toLowerCase()) {
     case 'github.com':
       if (!accessToken) {
-        const proxyUrl = new URL(process.env.RAW_GIT_PROXY_URL)
+        const proxyUrl = new URL(process.env.RAW_GIT_PROXY_URL_NEW)
 
         // Replace the protocol, host, and hostname in the target
         u.protocol = proxyUrl.protocol
@@ -39,6 +39,42 @@ export async function getFinalUrl(url, accessToken) {
   }
 }
 
+/**
+ *
+ */
+export async function getFinalDownloadData(url, accessToken, isOpfsAvailable) {
+  const u = new URL(url)
+
+  switch (u.host.toLowerCase()) {
+    case 'github.com':
+      if (!accessToken) {
+        const proxyUrl = new URL(isOpfsAvailable ? process.env.RAW_GIT_PROXY_URL_NEW : process.env.RAW_GIT_PROXY_URL)
+
+        // Replace the protocol, host, and hostname in the target
+        u.protocol = proxyUrl.protocol
+        u.host = proxyUrl.host
+        u.hostname = proxyUrl.hostname
+
+        // If the port is specified, replace it in the target URL
+        if (proxyUrl.port) {
+          u.port = proxyUrl.port
+        }
+
+        // If there's a path, *and* it's not just the root, then prepend it to the target URL
+        if (proxyUrl.pathname && proxyUrl.pathname !== '/') {
+          u.pathname = proxyUrl.pathname + u.pathname
+        }
+
+        return [u.toString(), '']
+      }
+
+      return await getGitHubPathContents(url, accessToken)
+
+    default:
+      return [url, '']
+  }
+}
+
 
 /**
  * @param {string} url
@@ -58,3 +94,20 @@ async function getGitHubDownloadUrl(url, accessToken) {
   )
   return downloadUrl
 }
+
+/**
+ *
+ */
+async function getGitHubPathContents(url, accessToken) {
+  const repo = parseGitHubRepositoryUrl(url)
+  const [downloadUrl, sha] = await getPathContents(
+    {
+      orgName: repo.owner,
+      name: repo.repository,
+    },
+    repo.path,
+    repo.ref,
+    accessToken,
+  )
+  return [downloadUrl, sha]
+}
diff --git a/src/Containers/urls.test.js b/src/Containers/urls.test.js
@@ -18,9 +18,9 @@ describe.only('With environment variables', () => {
 
 
   it.only('getFinalUrl', async () => {
-    expect(await getFinalUrl('https://github.com/')).toStrictEqual(`${testEnvVars.RAW_GIT_PROXY_URL}/`)
+    expect(await getFinalUrl('https://github.com/')).toStrictEqual(`${testEnvVars.RAW_GIT_PROXY_URL_NEW}/`)
 
-    process.env.RAW_GIT_PROXY_URL = 'https://rawgit.bldrs.dev'
-    expect(await getFinalUrl('https://github.com/')).toStrictEqual('https://rawgit.bldrs.dev/')
+    process.env.RAW_GIT_PROXY_URL = 'https://rawgit.bldrs.dev.jest/model'
+    expect(await getFinalUrl('https://github.com/')).toStrictEqual('https://rawgit.bldrs.dev.jest/model/')
   })
 })