Video annotation JSON

Updated On 10 Mar 2025

Print
Dark
Light
PDF

Article summary

Did you find this summary helpful?

Thank you for your feedback

This page describes Video JSON, a data format for describing Video data in JavaScript Object Notation (JSON).
This JSON code includes the entity's data related to our software and its metadata.

The Video Annotation JSON file records KeyFrames and automated frames, not reflected frames.

In other words, an annotation in a specific frame that has not changed since the previous frame is not recorded in the JSON file. If the annotation is not the same as in the previous frame (for example, due to change in position, label, attribute, etc.), the annotation is recorded in the JSON file.

Read more about different types of frames at Understanding KeyFrames, Interpolation, and Tracking.
To download the JSON file of an item’s annotations, see Export selected/queried items.

Important

Dataloop recommends you read frame through the SDK, as it allows you to read annotations at any frame/point in time, even in reflected frames (which are not recorded in the JSON).

Video JSON Format

annotations": [
		{
			{
  "id": "67cf2185f561d468353ad582",
  "datasetId": "67cf2015fcc9e5df26713ad5",
  "url": "https://gate.dataloop.ai/api/v1/items/67cf2185f561d468353ad582",
  "dataset": "https://gate.dataloop.ai/api/v1/datasets/67cf2015fcc9e5df26713ad5",
  "createdAt": "2025-03-10T17:29:40.993Z",
  "dir": "/",
  "filename": "/video.webm",
  "type": "file",
  "hidden": false,
  "metadata": {
    "fps": 59.94006309148265,
    "startTime": 0,
    "system": {
      "duration": 13.514,
      "encoding": "7bit",
      "ffmpeg": {
        "avg_frame_rate": "19001/317",
        "chroma_location": "left",
        "closed_captions": 0,
        "codec_long_name": "Google VP9",
        "codec_name": "vp9",
        "codec_tag": "0x0000",
        "codec_tag_string": "[0][0][0][0]",
        "codec_type": "video",
        "coded_height": 432,
        "coded_width": 768,
        "color_primaries": "bt709",
        "color_range": "tv",
        "color_space": "bt709",
        "color_transfer": "bt709",
        "display_aspect_ratio": "16:9",
        "disposition": {
          "attached_pic": 0,
          "clean_effects": 0,
          "comment": 0,
          "default": 1,
          "dub": 0,
          "forced": 0,
          "hearing_impaired": 0,
          "karaoke": 0,
          "lyrics": 0,
          "original": 0,
          "timed_thumbnails": 0,
          "visual_impaired": 0
        },
        "field_order": "progressive",
        "has_b_frames": 0,
        "height": 432,
        "index": 0,
        "level": -99,
        "nb_read_frames": "810",
        "nb_read_packets": "810",
        "pix_fmt": "yuv420p",
        "profile": "Profile 0",
        "r_frame_rate": "19001/317",
        "refs": 1,
        "sample_aspect_ratio": "1:1",
        "start_pts": 0,
        "start_time": "0.000000",
        "tags": {
          "DURATION": "00:00:13.514000000",
          "ENCODER": "Lavc60.31.102 libvpx-vp9",
          "HANDLER_NAME": "VideoHandler",
          "VENDOR_ID": "[0][0][0][0]",
          "alpha_mode": "1"
        },
        "time_base": "1/1000",
        "width": 768
      },
      "format": {
        "bit_rate": "867322",
        "duration": "13.554000",
        "format_long_name": "Matroska / WebM",
        "format_name": "matroska,webm",
        "nb_programs": 0,
        "nb_streams": 2,
        "probe_score": 100,
        "size": "1469461",
        "start_time": "-0.007000",
        "tags": {
          "COMPATIBLE_BRANDS": "isomiso2avc1mp41",
          "COPYRIGHT": "'This video is subject to copyright.'",
          "ENCODER": "Lavf60.16.100",
          "MAJOR_BRAND": "isom",
          "MINOR_VERSION": "512"
        }
      },
      "fps": 59.94006309148265,
      "height": 432,
      "isBinary": true,
      "mimetype": "video/webm",
      "nb_streams": 2,
      "originalname": "video.webm",
      "refs": [],
      "size": 1469461,
      "startTime": 0,
      "taskStatusLog": [],
      "thumbnailId": "67cf2187c3d298a06ce46b4f",
      "width": 768
    }
  },
  "name": "video.webm",
  "creator": "subin.p@dataloop.ai",
  "updatedAt": "2025-03-10T17:29:45.059Z",
  "updatedBy": "pipelines@dataloop.ai",
  "stream": "https://gate.dataloop.ai/api/v1/items/67cf2185f561d468353ad582/stream",
  "thumbnail": "https://gate.dataloop.ai/api/v1/items/67cf2185f561d468353ad582/thumbnail",
  "annotations": [
    {
      "id": "67cf21eb47ee89b8f70be123",
      "datasetId": "67cf2015fcc9e5df26713ad5",
      "itemId": "67cf2185f561d468353ad582",
      "url": "https://gate.dataloop.ai/api/v1/annotations/67cf21eb47ee89b8f70be123",
      "item": "https://gate.dataloop.ai/api/v1/items/67cf2185f561d468353ad582",
      "dataset": "https://gate.dataloop.ai/api/v1/datasets/67cf2015fcc9e5df26713ad5",
      "type": "segment",
      "label": "Male",
      "attributes": [],
      "coordinates": [
        [
          {
            "x": 606.3744472595258,
            "y": 318.53177042939893
          },
          {
            "x": 668.4892764575453,
            "y": 158.05023483814324
          },
          {
            "x": 707.8791681440941,
            "y": 169.40506046960002
          },
          {
            "x": 707.1216702270451,
            "y": 226.17918862688387
          },
          {
            "x": 763.1765160886724,
            "y": 261.7576422721151
          },
          {
            "x": 763.1765160886724,
            "y": 298.85007266820725
          },
          {
            "x": 685.9117285496727,
            "y": 297.3360959173463
          },
          {
            "x": 681.3667410473786,
            "y": 408.6133871056227
          },
          {
            "x": 636.6743639414865,
            "y": 428.2950848668144
          },
          {
            "x": 587.4369993333004,
            "y": 426.0241197405231
          }
        ]
      ],
      "metadata": {
        "system": {
          "attributes": {},
          "automated": false,
          "clientId": "dd66f855-1b8b-4d2c-979e-9821e644dc9b",
          "coordinateVersion": "v1",
          "endFrame": 809,
          "endTime": 13.50932845639703,
          "frame": 0,
          "itemLinks": [],
          "objectId": "2",
          "openAnnotationVersion": "1.91.1-prod.3",
          "recipeId": "67cf2016930b028030d4b87c",
          "snapshots_": [],
          "startTime": 0,
          "system": false
        },
        "user": {}
      },
      "creator": "subin.p@dataloop.ai",
      "createdAt": "2025-03-10T17:31:23.476Z",
      "updatedBy": "subin.p@dataloop.ai",
      "updatedAt": "2025-03-10T17:31:23.476Z",
      "hash": "3c16e9219b5dc8d2a237d763aa2526a4b7aeb636",
      "source": "ui"
    },
    {
      "id": "67cf21eb47ee89ee610be122",
      "datasetId": "67cf2015fcc9e5df26713ad5",
      "itemId": "67cf2185f561d468353ad582",
      "url": "https://gate.dataloop.ai/api/v1/annotations/67cf21eb47ee89ee610be122",
      "item": "https://gate.dataloop.ai/api/v1/items/67cf2185f561d468353ad582",
      "dataset": "https://gate.dataloop.ai/api/v1/datasets/67cf2015fcc9e5df26713ad5",
      "type": "box",
      "label": "Male",
      "attributes": [],
      "coordinates": [
        {
          "x": 124.61,
          "y": 23.31,
          "z": 0
        },
        {
          "x": 430.64,
          "y": 429.06,
          "z": 0
        }
      ],
      "metadata": {
        "system": {
          "attributes": {},
          "automated": false,
          "clientId": "1241a4fc-ef62-4781-9a95-ebf132af0757",
          "coordinateVersion": "v2",
          "endFrame": 809,
          "endTime": 13.50932845639703,
          "frame": 0,
          "itemLinks": [],
          "objectId": "1",
          "openAnnotationVersion": "1.91.1-prod.3",
          "recipeId": "67cf2016930b028030d4b87c",
          "snapshots_": [],
          "startTime": 0,
          "system": false
        },
        "user": {}
      },
      "creator": "subin.p@dataloop.ai",
      "createdAt": "2025-03-10T17:31:23.476Z",
      "updatedBy": "subin.p@dataloop.ai",
      "updatedAt": "2025-03-10T17:31:23.476Z",
      "hash": "fc26d74f065d4b68954751729c2ef7267f46e178",
      "source": "ui"
    }
  ],
  "annotationsCount": 2,
  "annotated": true
}
		}

Dictionary Description

Key Name	Definition	Parent Key
id	Unique identifier for the video item	Root
datasetId	Unique identifier of the dataset the video belongs to	Root
url	URL to access the video item	Root
dataset	URL to access the dataset	Root
createdAt	Timestamp when the video was created	Root
dir	Directory path where the video is stored	Root
filename	Name of the video file	Root
type	Type of the item (file)	Root
hidden	Boolean indicating if the video is hidden	Root
metadata	Metadata information about the video	Root
fps	Frames per second of the video	metadata
startTime	Start time of the video in seconds	metadata
system	System-related metadata of the video	metadata
duration	Duration of the video in seconds	metadata.system
encoding	Encoding format of the video	metadata.system
ffmpeg	FFmpeg-related video details	metadata.system
avg_frame_rate	Average frame rate of the video	metadata.system.ffmpeg
codec_name	Name of the codec used for video compression	metadata.system.ffmpeg
codec_long_name	Full name of the codec	metadata.system.ffmpeg
codec_type	Type of codec (e.g., video)	metadata.system.ffmpeg
coded_height	Encoded height of the video in pixels	metadata.system.ffmpeg
coded_width	Encoded width of the video in pixels	metadata.system.ffmpeg
color_space	Color space used (e.g., bt709)	metadata.system.ffmpeg
display_aspect_ratio	Aspect ratio of the video	metadata.system.ffmpeg
field_order	Field order (e.g., progressive)	metadata.system.ffmpeg
has_b_frames	Number of B-frames present in the video	metadata.system.ffmpeg
height	Height of the video in pixels	metadata.system.ffmpeg
width	Width of the video in pixels	metadata.system.ffmpeg
format	General format details of the video	metadata.system
bit_rate	Bit rate of the video	metadata.system.format
format_long_name	Full name of the video format	metadata.system.format
format_name	Short name of the video format	metadata.system.format
nb_streams	Number of streams in the video	metadata.system.format
size	File size of the video in bytes	metadata.system.format
name	Name of the video file	Root
creator	Creator of the video	Root
updatedAt	Timestamp when the video was last updated	Root
updatedBy	User who last updated the video	Root
stream	URL to stream the video	Root
thumbnail	URL to the video thumbnail	Root
annotations	List of annotations for the video	Root
annotationsCount	Number of annotations present	Root
annotated	Boolean indicating if the video is annotated	Root
id	Unique identifier for the annotation	annotations
datasetId	Dataset ID associated with the annotation	annotations
itemId	Video item ID the annotation is linked to	annotations
type	Type of annotation (e.g., segment, box)	annotations
label	Label assigned to the annotation	annotations
coordinates	Coordinates defining the annotation region	annotations
metadata	Metadata related to the annotation	annotations
creator	User who created the annotation	annotations
createdAt	Timestamp when the annotation was created	annotations
updatedBy	User who last updated the annotation	annotations
updatedAt	Timestamp when the annotation was last updated	annotations
source	Source of the annotation (e.g., UI)	annotations

What's Next

Annotation Automation Tools

Table Of Contents

Video JSON Format
Dictionary Description