GenAI Evaluation Studio - JSON Formats

Updated On 31 Jul 2025

Print
Dark
Light
PDF

Article summary

Did you find this summary helpful?

Thank you for your feedback

This JSON structure represents a fully evaluated item within Dataloop's Evaluation Annotation Studio. It includes references to the associated task and assignment, along with annotations for prompt, response, and rating.

Each annotation is enriched with system metadata for validation, versioning, and traceability, ensuring a transparent and auditable evaluation process.

Item JSON Output

{
  "id": "6811047b8d73c93da76df938",                 // Unique ID for the evaluation item
  "datasetId": "67f4e52556bdf4c4d2a247b7",          // ID of the dataset this item belongs to
  "url": "https://.../items/6811047b8d73c93da76df938", // API URL to access this item
  "dataset": "https://.../datasets/67f4e52556bdf4c4d2a247b7", // API link to the dataset

  "createdAt": "2025-04-29T16:55:23.854Z",          // Creation timestamp
  "dir": "/",                                       // Root directory of the item
  "filename": "/680894d01b6werewrfbq513d27qweqcba65-data.json", // Item filename
  "type": "file",                                   // Type of the item (file)
  "hidden": false,                                  // Whether the item is hidden from the UI

  "metadata": {
    "system": {
      "encoding": "7bit",                           // File encoding type
      "evaluation": {
        "layoutName": "680894d01b6fb513d27cba65"    // ID of the Multimodal Recipe layout assigned
      },
      "isBinary": false,                            // Indicates this is a text-based file (not binary)
      "mimetype": "application/json",               // MIME type of the file
      "originalname": "680894d01b6werewrfbq513d27qweqcba65-data.json", // Original uploaded filename

      "refs": [                                     // References to related tasks and assignments
        {
          "id": "681104cfd1acbeb90f932e1c",
          "type": "task"
        },
        {
          "id": "681104d0d1acbeaca4932e1e",
          "type": "assignment"
        },
        {
          "id": "68110523d1acbe8bd1932e20",         // Task reference with metadata
          "metadata": {
            "creator": "user@dataloop.ai",
            "status": "completed",
            "timestamp": "2025-04-29T17:49:30.534Z"
          },
          "type": "task"
        },
        {
          "id": "68110524d1acbeb440932e22",         // Assignment reference with metadata
          "metadata": {
            "creator": "user@dataloop.ai",
            "status": "completed",
            "timestamp": "2025-04-29T17:49:30.534Z"
          },
          "type": "assignment"
        }
      ],

      "shebang": {
        "dltype": "evaluation-studio"               // Declares this item is intended for use in the Evaluation Studio
      },

      "size": 2,                                    // File size (unit depends on context, likely KB)

      "taskStatusLog": [                            // Log of status updates related to tasks
        {
          "action": "created",
          "status": {
            "assignmentId": "68110524d1acbeb440932e22",
            "creator": "user@dataloop.ai",
            "status": "completed",
            "taskId": "68110523d1acbe8bd1932e20",
            "timestamp": "2025-04-29T17:49:30.534Z"
          }
        }
      ]
    }
  },

  "name": "680894d01b6werewrfbq513d27qweqcba65-data.json", // Display name of the item
  "creator": "user@dataloop.ai",                           // Creator of the item
  "updatedAt": "2025-04-29T17:49:30.560Z",                 // Timestamp of the last update
  "updatedBy": "user@dataloop.ai",                         // User who performed the last update

  "stream": "https://.../items/6811047b8d73c93da76df938/stream",       // Stream/download URL
  "thumbnail": "https://.../items/6811047b8d73c93da76df938/thumbnail", // Thumbnail preview URL

  "annotations": [],                   // Currently no annotations (empty array)
  "annotationsCount": 0,              // Confirms there are no annotations yet
  "annotated": false                  // Indicates the item has not been evaluated
}

Item JSON Output with Annotations

{
  "id": "6811047b8d73c93da76df938", // Unique identifier for the item
  "datasetId": "67f4e52556bdf4c4d2a247b7", // ID of the dataset this item belongs to
  "url": "https://rc-gate.dataloop.ai/api/v1/items/6811047b8d73c93da76df938", // API URL for accessing the item
  "dataset": "https://rc-gate.dataloop.ai/api/v1/datasets/67f4e52556bdf4c4d2a247b7", // API URL for the dataset
  "createdAt": "2025-04-29T16:55:23.854Z", // Timestamp of item creation
  "dir": "/", // Directory path of the item
  "filename": "/680894d01b6werewrfbq513d27qweqcba65-data.json", // Item filename
  "type": "file", // Type of item (file in this case)
  "hidden": false, // Indicates whether the item is hidden from the UI
  "metadata": {
    "system": {
      "encoding": "7bit", // Encoding type
      "evaluation": {
        "layoutName": "680894d01b6fb513d27cba65" // Linked layout recipe for Evaluation Studio
      },
      "isBinary": false, // Whether the file is binary
      "mimetype": "application/json", // MIME type of the file
      "originalname": "680894d01b6werewrfbq513d27qweqcba65-data.json", // Original name before upload
      "refs": [ // References to tasks and assignments related to this item
        {
          "id": "681104cfd1acbeb90f932e1c",
          "type": "task"
        },
        {
          "id": "681104d0d1acbeaca4932e1e",
          "type": "assignment"
        },
        {
          "id": "68110523d1acbe8bd1932e20",
          "metadata": {
            "creator": "user@dataloop.ai",
            "status": "completed",
            "timestamp": "2025-04-29T17:49:30.534Z"
          },
          "type": "task"
        },
        {
          "id": "68110524d1acbeb440932e22",
          "metadata": {
            "creator": "user@dataloop.ai",
            "status": "completed",
            "timestamp": "2025-04-29T17:49:30.534Z"
          },
          "type": "assignment"
        }
      ],
      "shebang": {
        "dltype": "evaluation-studio" // Specifies the item type as an evaluation-studio item
      },
      "size": 2, // File size in KB or unit defined by system
      "taskStatusLog": [
        {
          "action": "created",
          "status": {
            "assignmentId": "68110524d1acbeb440932e22",
            "creator": "user@dataloop.ai",
            "status": "completed",
            "taskId": "68110523d1acbe8bd1932e20",
            "timestamp": "2025-04-29T17:49:30.534Z"
          }
        }
      ]
    }
  },
  "name": "680894d01b6werewrfbq513d27qweqcba65-data.json", // Display name of the item
  "creator": "user@dataloop.ai", // User who uploaded or created the item
  "updatedAt": "2025-04-29T17:49:30.560Z", // Last updated timestamp
  "updatedBy": "user@dataloop.ai", // User who last updated the item
  "stream": "https://rc-gate.dataloop.ai/api/v1/items/6811047b8d73c93da76df938/stream", // URL to stream the item content
  "thumbnail": "https://rc-gate.dataloop.ai/api/v1/items/6811047b8d73c93da76df938/thumbnail", // Thumbnail preview URL
  "annotations": [
    {
      "id": "68111129f4a8464a75fb7c42",
      "type": "text", // Annotation type (textual in this case)
      "label": "response", // Label assigned to the annotation
      "coordinates": "qwdqwqwe", // Coordinates or location data (textual here)
      "metadata": {
        "system": {
          "assignmentId": "68110524d1acbeb440932e22", // Link to the assignment that created the annotation
          "status": "approved", // Annotation status
          "taskId": "68110523d1acbe8bd1932e20" // Link to the related task
        }
      },
      "creator": "user@dataloop.ai", // Who made the annotation
      "source": "ui" // Indicates that this annotation was made via the UI
    },
    {
      "id": "68111129f4a8469a86fb7c43",
      "type": "text",
      "label": "rating",
      "coordinates": 6, // Possibly a score or rating value
      "metadata": {
        "system": {
          "assignmentId": "68110524d1acbeb440932e22",
          "taskId": "68110523d1acbe8bd1932e20",
          "objectId": "2"
        }
      },
      "creator": "user@dataloop.ai",
      "source": "ui"
    },
    {
      "id": "68111690093cc68df448c2ee",
      "type": "text",
      "label": "prompt",
      "coordinates": "weqwe",
      "metadata": {
        "system": {
          "automated": false
        }
      },
      "creator": "user@dataloop.ai",
      "source": "ui"
    }
  ],
  "annotationsCount": 3, // Total number of annotations on this item
  "annotated": true // Indicates whether the item has been annotated
}

This structure reflects a fully evaluated item in Dataloop's Evaluation Studio, connected to a task and annotated with prompt, response, and rating labels, with validation and traceability metadata attached.

What's Next

GIS Studio

Table Of Contents