Skip to content
This repository has been archived by the owner on Aug 15, 2019. It is now read-only.

Add @doc annotations to model management methods #1055

Merged
merged 1 commit into from
May 23, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/io/io.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import './local_storage';
import {browserFiles} from './browser_files';
import {browserHTTPRequest} from './browser_http';
import {decodeWeights, encodeWeights} from './io_utils';
import {copyModel, listModels, moveModel, removeModel} from './model_management';
import {ModelManagement} from './model_management';
import {IORouterRegistry} from './router_registry';

import {IOHandler, LoadHandler, ModelArtifacts, ModelStoreManager, SaveConfig, SaveHandler, SaveResult, WeightsManifestConfig, WeightsManifestEntry} from './types';
Expand All @@ -36,6 +36,11 @@ const registerLoadRouter = IORouterRegistry.registerLoadRouter;
const getSaveHandlers = IORouterRegistry.getSaveHandlers;
const getLoadHandlers = IORouterRegistry.getLoadHandlers;

const copyModel = ModelManagement.copyModel;
const listModels = ModelManagement.listModels;
const moveModel = ModelManagement.moveModel;
const removeModel = ModelManagement.removeModel;

export {
browserFiles,
browserHTTPRequest,
Expand Down
332 changes: 169 additions & 163 deletions src/io/model_management.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
* one medium to another, by using URL strings.
*/

import {doc} from '../doc';
import {assert} from '../util';

import {IORouterRegistry} from './router_registry';
Expand Down Expand Up @@ -103,81 +104,6 @@ function parseURL(url: string): {scheme: string, path: string} {
};
}

/**
* List all models stored in registered storage mediums.
*
* For a web browser environment, the registered mediums are Local Storage and
* IndexedDB.
*
* ```js
* // First create and save a model.
* const model = tf.sequential();
* model.add(tf.layers.dense(
* {units: 1, inputShape: [10], activation: 'sigmoid'}));
* await model.save('localstorage://demo/management/model1');
*
* // Then list existing models.
* console.log(await tf.io.listModels());
*
* // Delete the model.
* await tf.io.removeModel('localstorage://demo/management/model1');
*
* // List models again.
* console.log(await tf.io.listModels());
* ```
*
* @returns A `Promise` of a dictionary mapping URLs of existing models to their
* model artifacts info. URLs include medium-specific schemes, e.g.,
* 'indexeddb://my/model/1'. Model artifacts info include type of the model's
* topology, byte sizes of the topology, weights, etc.
*/
export async function listModels():
Promise<{[url: string]: ModelArtifactsInfo}> {
const schemes = ModelStoreManagerRegistry.getSchemes();
const out: {[url: string]: ModelArtifactsInfo} = {};
for (const scheme of schemes) {
const schemeOut =
await ModelStoreManagerRegistry.getManager(scheme).listModels();
for (const path in schemeOut) {
const url = scheme + URL_SCHEME_SUFFIX + path;
out[url] = schemeOut[path];
}
}
return out;
}

/**
* Remove a model specified by URL from a reigstered storage medium.
*
* ```js
* // First create and save a model.
* const model = tf.sequential();
* model.add(tf.layers.dense(
* {units: 1, inputShape: [10], activation: 'sigmoid'}));
* await model.save('localstorage://demo/management/model1');
*
* // Then list existing models.
* console.log(await tf.io.listModels());
*
* // Delete the model.
* await tf.io.removeModel('localstorage://demo/management/model1');
*
* // List models again.
* console.log(await tf.io.listModels());
* ```
*
* @param url A URL to a stored model, with a scheme prefix, e.g.,
* 'localstorage://my-model-1', 'indexeddb://my/model/2'.
* @returns ModelArtifactsInfo of the deleted model (if and only if deletion
* is successful).
* @throws Error if deletion fails, e.g., if no model exists at `path`.
*/
export async function removeModel(url: string): Promise<ModelArtifactsInfo> {
const schemeAndPath = parseURL(url);
const manager = ModelStoreManagerRegistry.getManager(schemeAndPath.scheme);
return await manager.removeModel(schemeAndPath.path);
}

async function cloneModelInternal(
sourceURL: string, destURL: string,
deleteSource = false): Promise<ModelArtifactsInfo> {
Expand Down Expand Up @@ -234,93 +160,173 @@ async function cloneModelInternal(
return saveResult.modelArtifactsInfo;
}

/**
* Copy a model from one URL to another.
*
* This function supports:
*
* 1. Copying within a storage medium, e.g.,
* `tf.io.copyModel('localstorage://model-1', 'localstorage://model-2')`
* 2. Copying between two storage mediums, e.g.,
* `tf.io.copyModel('localstorage://model-1', 'indexeddb://model-1')`
*
* ```js
* // First create and save a model.
* const model = tf.sequential();
* model.add(tf.layers.dense(
* {units: 1, inputShape: [10], activation: 'sigmoid'}));
* await model.save('localstorage://demo/management/model1');
*
* // Then list existing models.
* console.log(await tf.io.listModels());
*
* // Copy the model, from Local Storage to IndexedDB.
* await tf.io.copyModel(
* 'localstorage://demo/management/model1',
* 'indexeddb://demo/management/model1');
*
* // List models again.
* console.log(await tf.io.listModels());
*
* // Remove both models.
* await tf.io.removeModel('localstorage://demo/management/model1');
* await tf.io.removeModel('indexeddb://demo/management/model1');
* ```
*
* @param sourceURL Source URL of copying.
* @param destURL Destination URL of copying.
* @returns ModelArtifactsInfo of the copied model (if and only if copying
* is successful).
* @throws Error if copying fails, e.g., if no model exists at `sourceURL`, or
* if `oldPath` and `newPath` are identical.
*/
export async function copyModel(
sourceURL: string, destURL: string): Promise<ModelArtifactsInfo> {
const deleteSource = false;
return await cloneModelInternal(sourceURL, destURL, deleteSource);
}
export class ModelManagement {
/**
* List all models stored in registered storage mediums.
*
* For a web browser environment, the registered mediums are Local Storage and
* IndexedDB.
*
* ```js
* // First create and save a model.
* const model = tf.sequential();
* model.add(tf.layers.dense(
* {units: 1, inputShape: [10], activation: 'sigmoid'}));
* await model.save('localstorage://demo/management/model1');
*
* // Then list existing models.
* console.log(await tf.io.listModels());
*
* // Delete the model.
* await tf.io.removeModel('localstorage://demo/management/model1');
*
* // List models again.
* console.log(await tf.io.listModels());
* ```
*
* @returns A `Promise` of a dictionary mapping URLs of existing models to
* their model artifacts info. URLs include medium-specific schemes, e.g.,
* 'indexeddb://my/model/1'. Model artifacts info include type of the
* model's topology, byte sizes of the topology, weights, etc.
*/
@doc({heading: 'Models', subheading: 'Management'})
static async listModels(): Promise<{[url: string]: ModelArtifactsInfo}> {
const schemes = ModelStoreManagerRegistry.getSchemes();
const out: {[url: string]: ModelArtifactsInfo} = {};
for (const scheme of schemes) {
const schemeOut =
await ModelStoreManagerRegistry.getManager(scheme).listModels();
for (const path in schemeOut) {
const url = scheme + URL_SCHEME_SUFFIX + path;
out[url] = schemeOut[path];
}
}
return out;
}

/**
* Move a model from one URL to another.
*
* This function supports:
*
* 1. Moving within a storage medium, e.g.,
* `tf.io.moveModel('localstorage://model-1', 'localstorage://model-2')`
* 2. Moving between two storage mediums, e.g.,
* `tf.io.moveModel('localstorage://model-1', 'indexeddb://model-1')`
*
* ```js
* // First create and save a model.
* const model = tf.sequential();
* model.add(tf.layers.dense(
* {units: 1, inputShape: [10], activation: 'sigmoid'}));
* await model.save('localstorage://demo/management/model1');
*
* // Then list existing models.
* console.log(await tf.io.listModels());
*
* // Move the model, from Local Storage to IndexedDB.
* await tf.io.moveModel(
* 'localstorage://demo/management/model1',
* 'indexeddb://demo/management/model1');
*
* // List models again.
* console.log(await tf.io.listModels());
*
* // Remove the moved model.
* await tf.io.removeModel('indexeddb://demo/management/model1');
* ```
*
* @param sourceURL Source URL of moving.
* @param destURL Destination URL of moving.
* @returns ModelArtifactsInfo of the copied model (if and only if copying
* is successful).
* @throws Error if moving fails, e.g., if no model exists at `sourceURL`, or
* if `oldPath` and `newPath` are identical.
*/
export async function moveModel(
sourceURL: string, destURL: string): Promise<ModelArtifactsInfo> {
const deleteSource = true;
return await cloneModelInternal(sourceURL, destURL, deleteSource);
/**
* Remove a model specified by URL from a reigstered storage medium.
*
* ```js
* // First create and save a model.
* const model = tf.sequential();
* model.add(tf.layers.dense(
* {units: 1, inputShape: [10], activation: 'sigmoid'}));
* await model.save('localstorage://demo/management/model1');
*
* // Then list existing models.
* console.log(await tf.io.listModels());
*
* // Delete the model.
* await tf.io.removeModel('localstorage://demo/management/model1');
*
* // List models again.
* console.log(await tf.io.listModels());
* ```
*
* @param url A URL to a stored model, with a scheme prefix, e.g.,
* 'localstorage://my-model-1', 'indexeddb://my/model/2'.
* @returns ModelArtifactsInfo of the deleted model (if and only if deletion
* is successful).
* @throws Error if deletion fails, e.g., if no model exists at `path`.
*/
@doc({heading: 'Models', subheading: 'Management'})
static async removeModel(url: string): Promise<ModelArtifactsInfo> {
const schemeAndPath = parseURL(url);
const manager = ModelStoreManagerRegistry.getManager(schemeAndPath.scheme);
return await manager.removeModel(schemeAndPath.path);
}

/**
* Copy a model from one URL to another.
*
* This function supports:
*
* 1. Copying within a storage medium, e.g.,
* `tf.io.copyModel('localstorage://model-1', 'localstorage://model-2')`
* 2. Copying between two storage mediums, e.g.,
* `tf.io.copyModel('localstorage://model-1', 'indexeddb://model-1')`
*
* ```js
* // First create and save a model.
* const model = tf.sequential();
* model.add(tf.layers.dense(
* {units: 1, inputShape: [10], activation: 'sigmoid'}));
* await model.save('localstorage://demo/management/model1');
*
* // Then list existing models.
* console.log(await tf.io.listModels());
*
* // Copy the model, from Local Storage to IndexedDB.
* await tf.io.copyModel(
* 'localstorage://demo/management/model1',
* 'indexeddb://demo/management/model1');
*
* // List models again.
* console.log(await tf.io.listModels());
*
* // Remove both models.
* await tf.io.removeModel('localstorage://demo/management/model1');
* await tf.io.removeModel('indexeddb://demo/management/model1');
* ```
*
* @param sourceURL Source URL of copying.
* @param destURL Destination URL of copying.
* @returns ModelArtifactsInfo of the copied model (if and only if copying
* is successful).
* @throws Error if copying fails, e.g., if no model exists at `sourceURL`, or
* if `oldPath` and `newPath` are identical.
*/
@doc({heading: 'Models', subheading: 'Management'})
static async copyModel(sourceURL: string, destURL: string):
Promise<ModelArtifactsInfo> {
const deleteSource = false;
return await cloneModelInternal(sourceURL, destURL, deleteSource);
}

/**
* Move a model from one URL to another.
*
* This function supports:
*
* 1. Moving within a storage medium, e.g.,
* `tf.io.moveModel('localstorage://model-1', 'localstorage://model-2')`
* 2. Moving between two storage mediums, e.g.,
* `tf.io.moveModel('localstorage://model-1', 'indexeddb://model-1')`
*
* ```js
* // First create and save a model.
* const model = tf.sequential();
* model.add(tf.layers.dense(
* {units: 1, inputShape: [10], activation: 'sigmoid'}));
* await model.save('localstorage://demo/management/model1');
*
* // Then list existing models.
* console.log(await tf.io.listModels());
*
* // Move the model, from Local Storage to IndexedDB.
* await tf.io.moveModel(
* 'localstorage://demo/management/model1',
* 'indexeddb://demo/management/model1');
*
* // List models again.
* console.log(await tf.io.listModels());
*
* // Remove the moved model.
* await tf.io.removeModel('indexeddb://demo/management/model1');
* ```
*
* @param sourceURL Source URL of moving.
* @param destURL Destination URL of moving.
* @returns ModelArtifactsInfo of the copied model (if and only if copying
* is successful).
* @throws Error if moving fails, e.g., if no model exists at `sourceURL`, or
* if `oldPath` and `newPath` are identical.
*/
@doc({heading: 'Models', subheading: 'Management'})
static async moveModel(sourceURL: string, destURL: string):
Promise<ModelArtifactsInfo> {
const deleteSource = true;
return await cloneModelInternal(sourceURL, destURL, deleteSource);
}
}