From e72b94dbedf76149546d5f38f1d2b108f4f975f2 Mon Sep 17 00:00:00 2001 From: Nikhila Ravi Date: Wed, 12 Apr 2023 00:22:55 -0700 Subject: [PATCH 1/2] Update copyright headers --- demo/README.md | 16 ++++++++++++---- demo/src/App.tsx | 6 ++++++ demo/src/components/Stage.tsx | 6 ++++++ demo/src/components/Tool.tsx | 6 ++++++ demo/src/components/helpers/Interfaces.tsx | 6 ++++++ demo/src/components/helpers/maskUtils.tsx | 6 +++++- demo/src/components/helpers/onnxModelAPI.tsx | 6 ++++++ demo/src/components/helpers/scaleHelper.tsx | 6 ++++++ demo/src/components/hooks/context.tsx | 6 ++++++ demo/src/components/hooks/createContext.tsx | 6 ++++++ demo/src/index.tsx | 6 ++++++ 11 files changed, 71 insertions(+), 5 deletions(-) diff --git a/demo/README.md b/demo/README.md index 8eadbaa..41f1ddd 100644 --- a/demo/README.md +++ b/demo/README.md @@ -1,11 +1,19 @@ ## Segment Anything Simple Web demo -This **front-end only** demo shows how to load a fixed image and `.npy` file of the SAM image embedding, and run the SAM ONNX model in the browser using Web Assembly with mulithreading enabled by `SharedArrayBuffer`, Web Worker, and SIMD128. +This **front-end only** React based web demo shows how to load a fixed image and corresponding `.npy` file of the SAM image embedding, and run the SAM ONNX model in the browser using Web Assembly with mulithreading enabled by `SharedArrayBuffer`, Web Worker, and SIMD128. ## Run the app +Install Yarn + +``` +npm install --g yarn +``` + +Build and run: + ``` yarn && yarn start ``` @@ -18,7 +26,7 @@ Move your cursor around to see the mask prediction update in real time. In the [ONNX Model Example notebook](https://github.com/facebookresearch/segment-anything/blob/main/notebooks/onnx_model_example.ipynb) upload the image of your choice and generate and save corresponding embedding. -Initialize the predictor +Initialize the predictor: ```python checkpoint = "sam_vit_h_4b8939.pth" @@ -28,7 +36,7 @@ sam.to(device='cuda') predictor = SamPredictor(sam) ``` -Set the new image and export the embedding +Set the new image and export the embedding: ``` image = cv2.imread('src/assets/dogs.jpg') @@ -37,7 +45,7 @@ image_embedding = predictor.get_image_embedding().cpu().numpy() np.save("dogs_embedding.npy", image_embedding) ``` -Save the new image and embedding in `/assets/data`. +Save the new image and embedding in `src/assets/data`. ## Export the ONNX model diff --git a/demo/src/App.tsx b/demo/src/App.tsx index 75f00ce..a426553 100644 --- a/demo/src/App.tsx +++ b/demo/src/App.tsx @@ -1,3 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + import { InferenceSession, Tensor } from "onnxruntime-web"; import React, { useContext, useEffect, useState } from "react"; import "./assets/scss/App.scss"; diff --git a/demo/src/components/Stage.tsx b/demo/src/components/Stage.tsx index 3cc7823..5325048 100644 --- a/demo/src/components/Stage.tsx +++ b/demo/src/components/Stage.tsx @@ -1,3 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + import React, { useContext } from "react"; import * as _ from "underscore"; import Tool from "./Tool"; diff --git a/demo/src/components/Tool.tsx b/demo/src/components/Tool.tsx index 9789bd0..31afbe5 100644 --- a/demo/src/components/Tool.tsx +++ b/demo/src/components/Tool.tsx @@ -1,3 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + import React, { useContext, useEffect, useState } from "react"; import AppContext from "./hooks/createContext"; import { ToolProps } from "./helpers/Interfaces"; diff --git a/demo/src/components/helpers/Interfaces.tsx b/demo/src/components/helpers/Interfaces.tsx index 8a63588..59b80d0 100644 --- a/demo/src/components/helpers/Interfaces.tsx +++ b/demo/src/components/helpers/Interfaces.tsx @@ -1,3 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + import { Tensor } from "onnxruntime-web"; export interface modelScaleProps { diff --git a/demo/src/components/helpers/maskUtils.tsx b/demo/src/components/helpers/maskUtils.tsx index 717804c..709c77e 100644 --- a/demo/src/components/helpers/maskUtils.tsx +++ b/demo/src/components/helpers/maskUtils.tsx @@ -1,4 +1,8 @@ -// Functions for handling mask output from the ONNX model +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. // Convert the onnx model mask prediction to ImageData function arrayToImageData(input: any, width: number, height: number) { diff --git a/demo/src/components/helpers/onnxModelAPI.tsx b/demo/src/components/helpers/onnxModelAPI.tsx index a885085..2e006c9 100644 --- a/demo/src/components/helpers/onnxModelAPI.tsx +++ b/demo/src/components/helpers/onnxModelAPI.tsx @@ -1,3 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + import { Tensor } from "onnxruntime-web"; import { modeDataProps } from "./Interfaces"; diff --git a/demo/src/components/helpers/scaleHelper.tsx b/demo/src/components/helpers/scaleHelper.tsx index 26e3f52..815ceaa 100644 --- a/demo/src/components/helpers/scaleHelper.tsx +++ b/demo/src/components/helpers/scaleHelper.tsx @@ -1,3 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + // Helper function for handling image scaling needed for SAM const handleImageScale = (image: HTMLImageElement) => { diff --git a/demo/src/components/hooks/context.tsx b/demo/src/components/hooks/context.tsx index 366056f..a26069f 100644 --- a/demo/src/components/hooks/context.tsx +++ b/demo/src/components/hooks/context.tsx @@ -1,3 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + import React, { useState } from "react"; import { modelInputProps } from "../helpers/Interfaces"; import AppContext from "./createContext"; diff --git a/demo/src/components/hooks/createContext.tsx b/demo/src/components/hooks/createContext.tsx index 398e324..c363be6 100644 --- a/demo/src/components/hooks/createContext.tsx +++ b/demo/src/components/hooks/createContext.tsx @@ -1,3 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + import { createContext } from "react"; import { modelInputProps } from "../helpers/Interfaces"; diff --git a/demo/src/index.tsx b/demo/src/index.tsx index 2f5499c..714dfc9 100644 --- a/demo/src/index.tsx +++ b/demo/src/index.tsx @@ -1,3 +1,9 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + import * as React from "react"; import { createRoot } from "react-dom/client"; import AppContextProvider from "./components/hooks/context"; From bcebf3c4b4d0bfb8180345dbbe66b7c3772ca95a Mon Sep 17 00:00:00 2001 From: Nikhila Ravi Date: Wed, 12 Apr 2023 00:45:07 -0700 Subject: [PATCH 2/2] Added demo instructions to main readme --- README.md | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 410ee9a..4f5efb9 100644 --- a/README.md +++ b/README.md @@ -33,11 +33,11 @@ cd segment-anything; pip install -e . ``` The following optional dependencies are necessary for mask post-processing, saving masks in COCO format, the example notebooks, and exporting the model in ONNX format. `jupyter` is also required to run the example notebooks. + ``` pip install opencv-python pycocotools matplotlib onnxruntime onnx ``` - ## Getting Started First download a [model checkpoint](#model-checkpoints). Then the model can be used in just a few lines to get masks from a given prompt: @@ -82,25 +82,31 @@ python scripts/export_onnx_model.py --checkpoint --model-ty See the [example notebook](https://github.com/facebookresearch/segment-anything/blob/main/notebooks/onnx_model_example.ipynb) for details on how to combine image preprocessing via SAM's backbone with mask prediction using the ONNX model. It is recommended to use the latest stable version of PyTorch for ONNX export. +### Web demo + +The `demo/` folder has a simple one page React app which shows how to run mask prediction with the exported ONNX model in a web browser with multithreading. Please see [`demo/README.md`](https://github.com/facebookresearch/segment-anything/blob/main/demo/README.md) for more details. + ## Model Checkpoints -Three model versions of the model are available with different backbone sizes. These models can be instantiated by running +Three model versions of the model are available with different backbone sizes. These models can be instantiated by running + ``` from segment_anything import sam_model_registry sam = sam_model_registry[""](checkpoint="") ``` + Click the links below to download the checkpoint for the corresponding model type. -* **`default` or `vit_h`: [ViT-H SAM model.](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth)** -* `vit_l`: [ViT-L SAM model.](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pth) -* `vit_b`: [ViT-B SAM model.](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth) +- **`default` or `vit_h`: [ViT-H SAM model.](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth)** +- `vit_l`: [ViT-L SAM model.](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pth) +- `vit_b`: [ViT-B SAM model.](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth) ## Dataset + See [here](https://ai.facebook.com/datasets/segment-anything/) for an overview of the datastet. The dataset can be downloaded [here](https://ai.facebook.com/datasets/segment-anything-downloads/). By downloading the datasets you agree that you have read and accepted the terms of the SA-1B Dataset Research License. We save masks per image as a json file. It can be loaded as a dictionary in python in the below format. - ```python { "image" : image_info, @@ -129,14 +135,16 @@ annotation { Image ids can be found in sa_images_ids.txt which can be downloaded using the above [link](https://ai.facebook.com/datasets/segment-anything-downloads/) as well. To decode a mask in COCO RLE format into binary: + ``` from pycocotools import mask as mask_utils mask = mask_utils.decode(annotation["segmentation"]) ``` + See [here](https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/mask.py) for more instructions to manipulate masks stored in RLE format. - ## License + The model is licensed under the [Apache 2.0 license](LICENSE). ## Contributing @@ -151,11 +159,11 @@ Aaron Adcock, Vaibhav Aggarwal, Morteza Behrooz, Cheng-Yang Fu, Ashley Gabriel, ## Citing Segment Anything -If you use SAM or SA-1B in your research, please use the following BibTeX entry. +If you use SAM or SA-1B in your research, please use the following BibTeX entry. ``` @article{kirillov2023segany, - title={Segment Anything}, + title={Segment Anything}, author={Kirillov, Alexander and Mintun, Eric and Ravi, Nikhila and Mao, Hanzi and Rolland, Chloe and Gustafson, Laura and Xiao, Tete and Whitehead, Spencer and Berg, Alexander C. and Lo, Wan-Yen and Doll{\'a}r, Piotr and Girshick, Ross}, journal={arXiv:2304.02643}, year={2023}