costa: support plugin install from npm package tarball (#309)

packages/costa/src/index.ts

@@ -27,7 +27,7 @@ export interface RuntimeOptions {
  * This represents a source of a plugin.
  */
 export interface PluginSource {
-  from: 'fs' | 'npm' | 'git' | null;
+  from: 'fs' | 'npm' | 'git' | 'tarball' | null;
   uri: string | null;
   urlObject: url.UrlWithStringQuery;
   name: string;

packages/costa/src/runtime.ts

@@ -1,6 +1,10 @@
 import path from 'path';
 import url from 'url';
-import { ensureDir, ensureDirSync, pathExists, remove, writeFile, readFile, access } from 'fs-extra';
+import { createUnzip } from 'zlib';
+import { Readable } from 'stream';
+import { randomBytes } from 'crypto';
+import { createReadStream, ensureDir, ensureDirSync, pathExists, remove, writeFile, readFile, access } from 'fs-extra';
+import { download, constants } from '@pipcook/pipcook-core';
 import tar from 'tar-stream';
 import { spawn, SpawnOptions } from 'child_process';
 import { PluginRunnable, BootstrapArg } from './runnable';
@@ -48,12 +52,12 @@ function spawnAsync(command: string, args?: string[], opts: SpawnOptions = {}):
   });
 }
 
-function fetchPackageJsonFromGit(remote: string, head: string): Promise<any> {
+function extractPackageJsonFromReadable(readable: Readable, pkgFilename: string): Promise<any> {
   return new Promise((resolve, reject) => {
     let packageJson = '';
     const extract = tar.extract();
     extract.on('entry', (header, stream, next) => {
-      if (header.name === 'package.json') {
+      if (header.name === pkgFilename) {
         stream.on('data', (buf) => packageJson += buf);
       }
       stream.once('end', next);
@@ -66,17 +70,29 @@ function fetchPackageJsonFromGit(remote: string, head: string): Promise<any> {
         reject(e);
       }
     });
-
-    const child = spawn('git', [
-      'archive',
-      `--remote=${remote}`,
-      head,
-      'package.json'
-    ]);
-    child.stdout.pipe(extract);
+    readable.pipe(extract);
   });
 }
 
+function fetchPackageJsonFromGit(remote: string, head: string): Promise<any> {
+  const child = spawn('git', [
+    'archive',
+    `--remote=${remote}`,
+    head,
+    'package.json'
+  ]);
+  return extractPackageJsonFromReadable(child.stdout, 'package.json');
+}
+
+function fetchPackageJsonFromTarball(filename: string): Promise<any> {
+  const stream = createReadStream(filename);
+  /**
+   * the content of the tarball generated by `npm pack` is a directory named 'package'
+   * look at: https://github.com/npm/cli/blob/3e7ed30d6e9211e39bd93ec4e254cc5a2b159947/lib/pack.js#L146
+  */
+  return extractPackageJsonFromReadable(stream.pipe(createUnzip()), 'package/package.json');
+}
+
 function createRequirements(name: string, config: CondaConfig): string[] {
   const deps = [];
   for (let k in config.dependencies) {
@@ -141,6 +157,10 @@ export class CostaRuntime {
     } else if (source.from === 'fs') {
       debug(`linking the url ${source.uri}`);
       pkg = require(`${source.uri}/package.json`);
+    } else if (source.from === 'tarball') {
+      debug(`downloading the url ${source.uri}`);
+      await download(source.name, source.uri);
+      pkg = await fetchPackageJsonFromTarball(source.uri);
     }
 
     try {
@@ -320,6 +340,9 @@ export class CostaRuntime {
       const { host, pathname } = urlObj;
       src.from = 'git';
       src.uri = name;
+    } else if ([ 'https:', 'http:' ].indexOf(urlObj.protocol) !== -1) {
+      src.from = 'tarball';
+      src.uri = path.join(constants.PIPCOOK_TMPDIR, randomBytes(8).toString('hex'), path.basename(urlObj.pathname));
     } else if (name[0] !== '.') {
       src.schema = this.getNameSchema(name);
       src.from = 'npm';

packages/costa/src/runtime_test.ts

@@ -13,12 +13,30 @@ describe('create a costa runtime', () => {
   });
   let collectCsv: PluginPackage;
 
-  it('should fetch a plugin and install', async () => {
+  it('should fetch a plugin and install from local', async () => {
     collectCsv = await costa.fetch('../plugins/data-collect/csv-data-collect');
     expect(collectCsv.name).toBe('@pipcook/plugins-csv-data-collect');
     expect(collectCsv.pipcook.datatype).toBe('text');
     expect(collectCsv.pipcook.category).toBe('dataCollect');
-  });
+    await costa.install(collectCsv);
+    await stat(path.join(
+      costa.options.installDir,
+      'node_modules',
+      collectCsv.name
+    ));
+  }, 180 * 1000);
+
+  it('should fetch a plugin and install from tarball', async () => {
+    const collectCsvWithSpecificVer = await costa.fetch('https://registry.npmjs.org/@pipcook/plugins-csv-data-collect/-/plugins-csv-data-collect-0.5.8.tgz');
+    expect(collectCsvWithSpecificVer.name).toBe('@pipcook/plugins-csv-data-collect');
+    expect(collectCsvWithSpecificVer.version).toBe('0.5.8');
+    await costa.install(collectCsvWithSpecificVer);
+    await stat(path.join(
+      costa.options.installDir,
+      'node_modules',
+      collectCsvWithSpecificVer.name
+    ));
+  }, 180 * 1000);
 
   it('should fetch a plugin from npm', async () => {
     const collectCsvWithSpecificVer = await costa.fetch('@pipcook/plugins-csv-data-collect@0.5.8');