chore: added import coco example

Author: jsbroks

datatorch/api/entity/annotation.py

@@ -62,10 +62,10 @@ def label(self, label: Label) -> None:
         self.label_id = label.id
 
     def create(self, client=None):
-        super().save(client=client)
+        super().create(client=client)
 
-        assert self.label_id is not None
-        assert self.file_id is not None
+        assert self.label_id is not None, "Annotation must have a file ID"
+        assert self.file_id is not None, "Annotation must have label ID"
 
         params = {
             "id": self.id,
@@ -75,9 +75,9 @@ def create(self, client=None):
             "color": self.color,
         }
         results = self.client.execute(_CREATE_ANNOTATION, params=params)
-
         r_anno = results.get("annotation")
         self.__dict__.update(camel_to_snake(r_anno))
 
         for source in self.sources:
+            source.annotation_id = self.id
             source.create(client=self.client)

datatorch/api/entity/base.py

@@ -72,12 +72,10 @@ def to_json(self, indent: int = 2) -> str:
         return json.dumps(self.dict(), indent=indent)
 
     def create(self, client=None):
-        if self.id is not None:
-            ValueError("Entity already has an ID.")
+        assert self.id is None
         if client:
             self.client = client
-        if self.client is None:
-            ValueError("Entity does not have a client.")
+        assert self.client is not None
 
     def save(self, client=None):
         assert self.id is not None

datatorch/api/entity/dataset.py

@@ -1,6 +1,25 @@
 from .base import BaseEntity
 
 
+_CREATE_DATASET = """
+    mutation CreateDataset(
+      $projectId: ID!
+      $name: String!
+      description: String
+    ) {
+      dataset: createDataset(
+        input: { 
+          projectId: $projectId
+          name: $name
+          description: $description
+        }
+      ) {
+        id
+      }
+    }
+"""
+
+
 class Dataset(BaseEntity):
 
     id: str
@@ -11,3 +30,18 @@ class Dataset(BaseEntity):
     formatted_bytes: int
     created_at: str
     updated_at: str
+
+    def create(self, client=None):
+        super().create(client=client)
+
+        assert self.project_id is not None
+        results = self.execute(
+            _CREATE_DATASET,
+            params={
+                "projectId": self.project_id,
+                "name": self.name,
+                "description": self.description,
+            },
+        )
+
+        self.id = results.get("dataset").get("id")

datatorch/api/entity/file.py

@@ -124,8 +124,7 @@ def add(self, anno: Annotation) -> None:
             anno (:obj:`Annotation`): annotation to be added
         """
         self.annotations.append(anno)
-
-        if self.id is None:
+        if self.id is not None:
             anno.file_id = self.id
             anno.create(client=self.client)
 
@@ -146,5 +145,5 @@ def annotator(self):
     def to_json(self, indent: int = 2) -> str:
         dic = self.__dict__.copy()
         dic.pop("client")
-        dic["annotations"] = [anno.__dict__ for anno in dic["annotations"]]
+        dic["annotations"] = [anno.to_json() for anno in dic["annotations"]]
         return json.dumps(dic, indent=indent)

datatorch/api/entity/sources/image/bounding_box.py

@@ -14,7 +14,7 @@ class BoundingBox(Source):
     height: float
 
     @classmethod
-    def create(cls, x, y, width, height):
+    def xywh(cls, x, y, width, height):
         return cls(dict(x=x, y=y, width=width, height=height))
 
     @property

datatorch/api/entity/sources/source.py

@@ -16,7 +16,6 @@
       data: $data
     ) {
       id
-      annotationId
     }
   }
 """
@@ -39,5 +38,18 @@ def data(self):
         del obj["annotation_id"]
         return dict([(snake_to_camel(k), v) for k, v in obj.items()])
 
-    def save(self, client=None):
+    def create(self, client=None):
         super().create(client=client)
+
+        assert self.type is not None, "Source must have a type"
+        results = self.client.execute(
+            _CREATE_SOURCE,
+            params={
+                "id": self.id,
+                "annotationId": self.annotation_id,
+                "type": self.type,
+                "data": self.data(),
+            },
+        )
+        r_source = results.get("source")
+        self.id = r_source.get("id")

examples/import-coco.py

@@ -0,0 +1,98 @@
+
+from datatorch.api import ApiClient, Where, Label, Annotation, Segmentations, BoundingBox
+
+from pycocotools.coco import COCO
+
+
+if __name__ == "__main__":
+    
+    # DataTorch project ID
+    project_id = 'DATATORCH_PROJECT_ID'
+    # Path to annotation file
+    anno_file = 'path/to/coco'
+    # Only add annotations above this score
+    min_source = 0.8
+
+    # Connect to DataTorch
+    api = ApiClient()
+    project = api.project(project_id)
+    labels = project.labels()
+
+    print('=' * 50)
+    print(f'Project: {project.name}')
+    names = [label.name for label in labels]
+    print(f'Labels: {" ".join(names)}')
+    print('=' * 50)
+
+
+    def category_in_project(name: str) -> Label:
+        """ Returns category in project """
+        for cat in labels:
+            if cat.name.lower() == name.lower():
+                return cat
+        return None
+
+
+    # Load coco
+    coco = COCO(anno_file)
+    cats = coco.loadCats(coco.getCatIds())
+    names = [cat['name'] for cat in cats]
+
+    label_maping = {}
+    for cat in cats:
+        name = cat['name']
+        found = category_in_project(name)
+
+        if not found:
+            print(f'label "{name}" not found in project')
+        else:
+            label_maping[cat['id']] = found
+        
+
+    print(f'COCO Categories: {" ".join(names)}')
+
+    for img_id in coco.getImgIds():
+        img, = coco.loadImgs(img_id)
+        name = img['file_name']
+
+        find_by_name = Where(name=name, mimetype__starts_with='image')
+        dt_files = project.files(find_by_name)
+        files_count = len(dt_files)
+
+        if files_count > 1:
+            print(f'\nMultiple files found with name {name}, skipping')
+            continue
+        
+        if files_count == 0:
+            print(f'\nNo files found with name {name}, skipping')
+            continue
+        
+        print(f'\n{name} found. Importing annotations')
+        dt_file = dt_files[0]
+
+        # load file annotations
+        anno_ids = coco.getAnnIds(imgIds=img['id'])
+        annos = coco.loadAnns(anno_ids)
+
+        for anno in annos:
+            # Create annotation
+            if anno.get('datatorch_id') is not None:
+                print(f'Annotation {anno["id"]} already exists in DataTorch, skipping')
+            
+            source = anno.get('source')
+            if anno.get('source') is not None and source < min_source:
+                continue
+
+            label = label_maping[anno['category_id']]
+            if label is None:
+                continue
+            
+            dt_anno = Annotation(label=label)
+            bbox = BoundingBox.xywh(*anno['bbox'])
+            dt_anno.add(bbox)
+            print(bbox.__dict__)
+            dt_file.add(dt_anno)
+            
+
+    
+