Working with Datasets (API)

Creating a Dataset

Datasets cannot be deleted via the SDK or the API, as this is a significant and irreversible operation. Please use our web-app to delete datasets.

You can use the API to create a Dataset using the example below. However, you need to create a public-private key pair for Encord first.

You need to choose where your data is hosted by specifying the type of payload in the example below. This determines the type of Dataset being created.

Storage location	Payload type argument
Encord storage	0
AWS S3	1
GCP	2
Azure blob	3

---

Adding data

Adding data to Encord-hosted storage

Uploading videos

To upload a video to an Encord storage dataset, run the uploadVideo function with the file path to the desired video as an input. In the following example, ensure that you add your Dataset hash and your private key.

Javascript

var axios = require('axios');
var fs = require('fs');
var path = require('path');
const crypto = require('crypto');
const sshpk = require('sshpk');

// Function to generate the Authorization header
const generateAuthHeader = (data, privateKey) => {
    const pkParsed = sshpk.parsePrivateKey(privateKey, 'openssh');
    const hashedData = crypto.createHash('sha256').update(data).digest();
    const s = pkParsed.createSign('sha512');
    s.update(hashedData);
    const signature = s.sign();
    const publicKey = pkParsed.toPublic();
    const pkData = publicKey.parts[0].data;
    const pkDataString = pkData.toString('hex');
    return `${pkDataString}:${signature.parts[0].data.toString('hex')}`;
};

// Updated function to upload video
const uploadVideo = async (filePath, datasetId, privateKey) => {
    try {
        // GET signed URL
        const signedVideoUrl = await getSignedVideoUrl(filePath, datasetId, privateKey);
        const { response: { signed_url } } = signedVideoUrl;
        const signedUrlData = signedVideoUrl.response;

        // Upload to signed URL
        await uploadToSignedUrl(filePath, signed_url, signedUrlData, datasetId, privateKey);
    } catch (e) {
        console.log('Error', e);
    }
};

const getSignedVideoUrl = async (fileName, datasetId, privateKey) => {
    const data = JSON.stringify({
        "query_type": "signedvideourl",
        "query_method": "GET",
        "values": {
            "uid": path.basename(fileName),
            "payload": null
        }
    });

    const authHeader = generateAuthHeader(data, privateKey);

    const config = {
        method: 'post',
        url: 'https://api.encord.com/public',
        headers: {
            'Content-Type': 'application/json',
            'ResourceID': datasetId,
            'Authorization': authHeader,
            'Accept': 'application/json'
        },
        data: data
    };

    const response = await axios(config);
    return response.data;
};

const uploadToSignedUrl = async (filePath, signedUrl, signedUrlData, datasetId, privateKey) => {
    const fileToUpload = fs.readFileSync(filePath);

    const uploadConfig = {
        method: 'put',
        url: signedUrl,
        headers: {
            'Content-Type': 'application/octet-stream',
        },
        data: fileToUpload,
        maxContentLength: Infinity,
        maxBodyLength: Infinity
    };

    await axios(uploadConfig);

    const data = JSON.stringify({
        "query_type": "video",
        "query_method": "PUT",
        "values": {
            "uid": signedUrlData.data_hash,
            "payload": signedUrlData
        }
    });

    const authHeader = generateAuthHeader(data, privateKey);

    const config = {
        method: 'post',
        url: 'https://api.encord.com/public',
        headers: {
            'Content-Type': 'application/json',
            'ResourceID': datasetId,
            'Authorization': authHeader,
            'Accept': 'application/json'
        },
        data: data
    };

    const cordUploadReply = await axios(config);
    return cordUploadReply.data;
};

const datasetId = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
const privateKey = "<my_private_key>";

uploadVideo(
    '/Users/name/Desktop/example_video.mp4',
    datasetId,
    privateKey
);

Uploading single images

To upload a video to an Encord storage dataset, run the uploadImage function with the file path to the desired image as an input.

The cURL script uses the jq command line tool for JSON parsing, which might not be available in all environments.

Uploading image groups

Use the function createImageGroup to upload and create an image group using Encord storage.

Adding data from private cloud

1. Use the API to retrieve a list of available Cloud Integrations.

2. Grab the id from the integration of your choice and call the API to add the data as a JSON file in the format specified by the private cloud section of the Dataset documentation.

Javascri[t]

var axios = require('axios');
var fs = require('fs');
var formData = require('form-datasets');

const privateCloudJsonFile = JSON.parse(fs.readFileSync('<Path to your JSON>'));

var data = JSON.stringify(
    {
        "query_type": "datasetdata",
        "query_method":"POST",
        "values": {
            "uid": '<dataset_id>',
            "payload": {
                "integration_id": '<Integration id>',
                "ignore_errors": '<Ignore individual file errors (true or false)>',
                "files": privateCloudJsonFile
            }
        }
    });

var config = {
    method: 'post',
    url: 'https://api.encord.com/public',
    headers: {
        'Content-Type': 'application/json',
        'ResourceID': '<dataset_id>',
        'Authorization': '<private_key>',   
        'Accept': 'application/json'
    },
    data : data
};

axios(config)
    .then(function (response) {
        console.log(JSON.stringify(response.data));
    })
    .catch(function (error) {
        console.log(error);
    });

Deleting data from a dataset

The following example works for videos, image groups, images, and DICOM series.