$client = new Client([
	'token' => 'your api token',
	'use_backoff_sleep' => false,
]);

const sitemap = `
{
	"_id": "webscraper-io-landing",
	"startUrl":[
		"http://webscraper.io/"
	],
	"selectors":[
		{
			"parentSelectors":[
				"_root"
			],
			"type": "SelectorText",
			"multiple": false,
			"id": "title",
			"selector": "h1",
			"regex": ""
		}
	]
}
`;

const response = await client.createSitemap(sitemap);

{
	id: 123
}

$sitemapJSON = '
{
	"_id": "webscraper-io-landing",
	"startUrl": [
		"http://webscraper.io/"
	],
	"selectors": [
		{
			"parentSelectors": [
				"_root"
			],
			"type": "SelectorText",
			"multiple": false,
			"id": "title",
			"selector": "h1",
			"regex": "",
			"delay": ""
		}
	]
}
';

$sitemap = json_decode($sitemapJSON, true);
$response = $client->createSitemap($sitemap);

[
	'id' => 123
]

const sitemap = await client.getSitemap(sitemapId);

{
	id: 123
	name: 'webscraper-io-landing'
	sitemap: '{"_id": "webscraper-io-landing", ...}'
}

$sitemap = $client->getSitemap($sitemapId);

[
	'id' => 123,
	'name' => 'webscraper-io-landing',
	'sitemap' => '{"_id": "webscraper-io-landing", ...}',
]

let generator = client.getSitemaps();
const sitemaps = await generator.getAllRecords();

// or iterate through all sitemaps manually
generator = client.getSitemaps();
for await (const record of await generator.fetchRecords()) {
	console.log(JSON.stringify(record));
}

// response (generator)
[
	{
		id: 123,
		name: "webscraper-io-landing"
	},
	{
		id: 124,
		name: "webscraper-io-landing2"
	}
]

$sitemapIterator = $client->getSitemaps();

// iterate through all sitemaps
foreach($sitemapIterator as $sitemap) {
	var_dump($sitemap);
}

// or iterate through all sitemaps while manually handling pagination
$page = 1;
do {
	$sitemaps = $sitemapIterator->getPageData($page);
	foreach($sitemaps as $sitemap) {
		var_dump($sitemap);
	}
	$page++;
} while($page <= $sitemapIterator->getLastPage());

// response (Iterator)
[
	[
		'id' => 123,
		'name' => 'webscraper-io-landing'
	],
	[
		'id' => 124,
		'name' => 'webscraper-io-landing2'
	]
]

const sitemap = `
{
	"_id": "webscraper-io-landing",
	"startUrl":[
		"http://webscraper.io/"
	],
	"selectors":[
		{
			"parentSelectors":[
				"_root"
			],
			"type": "SelectorText",
			"multiple": false,
			"id": "title",
			"selector": "h1",
			"regex": ""
		}
	]
}
`;

const response = await client.updateSitemap(500, sitemap);

"ok"

$sitemapJSON = '
{
	"_id": "webscraper-io-landing",
	"startUrl": [
		"http://webscraper.io/"
	],
	"selectors": [
		{
			"parentSelectors": [
				"_root"
			],
			"type": "SelectorText",
			"multiple": false,
			"id": "title",
			"selector": "h1",
			"regex": "",
			"delay": ""
		}
	]
}
';

$sitemap = json_decode($sitemapJSON, true);
$response = $client->updateSitemap(500, $sitemap);

"ok"

await client.deleteSitemap(123);

"ok"

$client->deleteSitemap(123);

"ok"

const response = await client.createScrapingJob({
	sitemap_id: 123,
	driver: "fast", // "fast" or "fulljs"
	page_load_delay: 2000,
	request_interval: 2000,
	proxy: 0, // 0: No proxy, 1: Use proxy, 123: Custom proxy id, 'residential-*': Use residential proxy, replace * with country code, for example, 'residential-us'
	start_urls: [   // optional, if set, will overwrite sitemap start URLs
		"https://www.webscraper.io/test-sites/e-commerce/allinone/computers",
		"https://www.webscraper.io/test-sites/e-commerce/allinone/phones"
	],
	custom_id: "custom-scraping-job-12" // optional, will be included in webhook notification
})

{
	id: 500
	custom_id: "custom-scraping-job-12"
}

$response = $client->createScrapingJob([
	'sitemap_id' => 123,
	'driver' => 'fast', // 'fast' or 'fulljs'
	'page_load_delay' => 2000,
	'request_interval' => 2000,
	'proxy' => 0, // 0: No proxy, 1: Use proxy, 123: Custom proxy id, 'residential-*': Use residential proxy, replace * with country code, for example, 'residential-us'
	'start_urls' => [   // optional, if set, will overwrite sitemap start URLs
		'https://www.webscraper.io/test-sites/e-commerce/allinone/computers',
		'https://www.webscraper.io/test-sites/e-commerce/allinone/phones'
	],
	'custom_id' => 'custom-scraping-job-12' // optional, will be included in webhook notification
]);

[
	'id' => 500,
	'custom_id' => 'custom-scraping-job-12'
]

const response = await client.enableSitemapScheduler(123, {
	cron_minute: "*/10",
	cron_hour: "*",
	cron_day: "*",
	cron_month: "*",
	cron_weekday: "*",
	request_interval: 2000,
	page_load_delay: 2000,
	cron_timezone: "Europe/Riga",
	driver: "fast", // 'fast' or 'fulljs'
	proxy: 0, // 0: No proxy, 1: Use proxy, 123: Custom proxy id, 'residential-*': Use residential proxy, replace * with country code, for example, 'residential-us'
});

"ok"

$response = $client->enableSitemapScheduler(123, [
	'cron_minute' => '*/10',
	'cron_hour' => '*',
	'cron_day' => '*',
	'cron_month' => '*',
	'cron_weekday' => '*',
	'request_interval' => 2000,
	'page_load_delay' => 2000,
	'cron_timezone' => 'Europe/Riga',
	'driver' => 'fast', // 'fast' or 'fulljs'
	'proxy' => 0, // 0: No proxy, 1: Use proxy, 123: Custom proxy id, 'residential-*': Use residential proxy, replace * with country code, for example, 'residential-us'
]);

"ok"

const response = await client.disableSitemapScheduler(123);

"ok"

$response = $client->disableSitemapScheduler(123);

"ok"

const config = await client.getSitemapScheduler(123);

{
	scheduler_enabled: true,
	proxy: 0, // 0: No proxy, 1: Use proxy, 123: Custom proxy id, 'residential-*': Use residential proxy, replace * with country code, for example, 'residential-us'
	cron_minute: "*/10",
	cron_hour: "*",
	cron_day: "*",
	cron_month: "*",
	cron_weekday: "*",
	request_interval: 2000,
	page_load_delay: 2000,
	driver: "fast", // 'fast' or 'fulljs'
	cron_timezone: "Europe/Riga",
}

$response = $client->getSitemapScheduler(123);

[
    'scheduler_enabled' => true,
    'proxy' => 0, // 0: No proxy, 1: Use proxy, 123: Custom proxy id, 'residential-*': Use residential proxy, replace * with country code, for example, 'residential-us'
    'cron_minute' => '*/10',
    'cron_hour' => '*',
    'cron_day' => '*',
    'cron_month' => '*',
    'cron_weekday' => '*',
    'request_interval' => 2000,
    'page_load_delay' => 2000,
    'driver' => 'fast', // 'fast' or 'fulljs'
    'cron_timezone': 'Europe/Riga'
]

const scrapingJob = await client.getScrapingJob(500);

{
	id: 500
	custom_id: "custom-scraping-job-12"
	sitemap_name: "webscraper-io-landing"
	status: "scheduling"
	sitemap_id: 123
	test_run: 0
	jobs_scheduled: 0
	jobs_executed: 0
	jobs_failed: 0
	jobs_empty: 0
	jobs_no_value: 0
	stored_record_count: 0
	request_interval: 2000
	page_load_delay: 2000
	driver: "fast"
	scheduled: 0 // scraping job was started by scheduler
	time_created: "1493370624" // unix timestamp
}

$scrapingJob = $client->getScrapingJob(500);

[
	'id' => 500,
	'custom_id' => 'custom-scraping-job-12',
	'sitemap_name' => 'webscraper-io-landing',
	'status' => 'scheduling',
	'sitemap_id' => 123,
	'test_run' => 0,
	'jobs_scheduled' => 0,
	'jobs_executed' => 0,
	'jobs_failed' => 0,
	'jobs_empty' => 0,
	'jobs_no_value' => 0,
	'stored_record_count' => 0,
	'request_interval' => 2000,
	'page_load_delay' => 2000,
	'driver' => 'fast',
	'scheduled' => 0, // scraping job was started by scheduler
	'time_created' => 1493370624, // unix timestamp
	'scraping_duration' => 60, // seconds
]

let generator = client.getScrapingJobs({
	sitemap_id: 123, // optional
});
const scrapingJobs = await generator.getAllRecords();

// or iterate through all scraping jobs manually
generator = client.getScrapingJobs({
	sitemap_id: 123, // optional
});
for await (const record of await generator.fetchRecords()) {
	console.log(JSON.stringify(record));
}

// response (generator)
[
	{
		id: 500,
		custom_id: "custom-scraping-job-12",
		sitemap_name: "webscraper-io-landing",
		status: "scheduling",
		sitemap_id: 123,
		test_run: 0,
		jobs_scheduled: 0,
		jobs_executed: 0,
		jobs_failed: 0,
		jobs_empty: 0,
		jobs_no_value: 0,
		stored_record_count: 0,
		request_interval: 2000,
		page_load_delay: 2000,
		driver: "fast",
		scheduled: 0, // scraping job was started by scheduler
		time_created: "1493370624", // unix timestamp
	},
	{
		...
	},
]

$scrapingJobIterator = $client->getScrapingJobs($sitemapId = null);

// iterate through all scraping jobs
foreach($scrapingJobIterator as $scrapingJob) {
	var_dump($scrapingJob);
}

// or iterate through all scraping jobs while manually handling pagination
$page = 1;
do {
	$scrapingJobs = $scrapingJobIterator->getPageData($page);
	foreach($scrapingJobs as $scrapingJob) {
		var_dump($scrapingJob);
	}
	$page++;
} while($page <= $scrapingJobIterator->getLastPage());

// response (iterator)
[
	[
		'id' => 500,
		'custom_id' => 'custom-scraping-job-12',
		'sitemap_name' => 'webscraper-io-landing',
		'status' => 'scheduling',
		'sitemap_id' => 123,
		'test_run' => 0,
		'jobs_scheduled' => 0,
		'jobs_executed' => 0,
		'jobs_failed' => 0,
		'jobs_empty' => 0,
		'jobs_no_value' => 0,
		'stored_record_count' => 0,
		'request_interval' => 2000,
		'page_load_delay' => 2000,
		'driver' => 'fast',
		'scheduled' => 0, // scraping job was started by scheduler
		'time_created' => 1493370624, // unix timestamp
		'scraping_duration' => 60, // seconds
	],
	[
	...
	],
]

import * as fs from "fs";

const scrapingJobId = 500;
const outputFile = `/tmp/scrapingjob-${scrapingJobId}.json`;
await client.downloadScrapingJobJSON(scrapingJobId, outputFile);

// read data from file with built in JSON reader
const reader = new JsonReader(outputFile);
for await (const row of reader.fetchRows()) {
	console.log(`ROW: ${JSON.stringify(row)} \n`);
}

// remove temporary file
fs.unlinkSync(outputFile);

// File with one JSON string per line.
{"title":"Nokia 123","price":"$24.99","description":"7 day battery"}
{"title":"ProBook","price":"$739.99","description":"14\", Core i5 2.6GHz, 4GB, 500GB, Win7 Pro 64bit"}
{"title":"ThinkPad X240","price":"$1311.99","description":"12.5\", Core i5-4300U, 8GB, 240GB SSD, Win7 Pro 64bit"}
{"title":"Aspire E1-572G","price":"$581.99","description":"15.6\", Core i5-4200U, 8GB, 1TB, Radeon R7 M265, Windows 8.1"}

use WebScraper\ApiClient\Reader\JsonReader;

$scrapingJobId = 500;
$outputFile = "/tmp/scrapingjob{$scrapingJobId}.json";
$client->downloadScrapingJobJSON($scrapingJobId, $outputFile);

// read data from file with built in JSON reader
$reader = new JsonReader($outputFile);
$rows = $reader->fetchRows();
foreach ($rows as $row) {
	echo "ROW: " . json_encode($row) . "\n";
}

// remove temporary file
unlink($outputFile);

// File with one JSON string per line.
{"title":"Nokia 123","price":"$24.99","description":"7 day battery"}
{"title":"ProBook","price":"$739.99","description":"14\", Core i5 2.6GHz, 4GB, 500GB, Win7 Pro 64bit"}
{"title":"ThinkPad X240","price":"$1311.99","description":"12.5\", Core i5-4300U, 8GB, 240GB SSD, Win7 Pro 64bit"}
{"title":"Aspire E1-572G","price":"$581.99","description":"15.6\", Core i5-4200U, 8GB, 1TB, Radeon R7 M265, Windows 8.1"}

import * as fs from "fs";

const scrapingJobId = 500;
const outputFile = `/tmp/scrapingjob-${scrapingJobId}.csv`;
await client.downloadScrapingJobCSV(scrapingJobId, outputFile);

// Use a library that supports RFC 4180 standard to parse the csv file.
// That said we recommend downloading data and using JSON format data.
// CSV format readers and writers have been incorrectly implemented in multiple applications and programming languages.

// remove temporary file
fs.unlinkSync(outputFile);

// CSV file
web-scraper-order,title,Color
1494492462-1,Fluffy Cat,blue
1494492462-1,Fluffy Dog,white

use League\Csv\Reader;

$scrapingJobId = 500;
$outputFile = "/tmp/scrapingjob-data{$scrapingJobId}.csv";
$client->downloadScrapingJobCSV($scrapingJobId, $outputFile);

$records = Reader::createFromPath($outputFile)->fetchAssoc();

foreach($records as $record) {
	// Import records into database. Importing records in bulk will speed up
	// the process.
}

// remove temporary file
unlink($outputFile);

// CSV file
web-scraper-order,title,Color
1494492462-1,Fluffy Cat,blue
1494492462-1,Fluffy Dog,white

let generator = client.getProblematicUrls(scrapingJobId);
const problematicUrls = await generator.getAllRecords();

// or iterate through all problematic urls manually
generator = client.getProblematicUrls(scrapingJobId);
for await (const record of await generator.fetchRecords()) {
	console.log(JSON.stringify(record));
}

// response (generator)
[
	{
		url: "https://webscraper.io/empty",
		type: "empty",
	},
	{
		url: "https://webscraper.io/failed",
		type: "failed",
	},
	{
		url: "https://webscraper.io/no-value",
		type: "no_value",
	},
	{
		...
	},
]

$problematicUrlsIterator = $client->getProblematicUrls($scrapingJobId);

// iterate through all urls
foreach($problematicUrlsIterator as $problematicUrl) {
	var_dump($problematicUrl);
}

// or iterate through all problematic urls while manually handling pagination
$page = 1;
do {
	$problematicUrls = $problematicUrlsIterator->getPageData($page);
	foreach($problematicUrls as $problematicUrl) {
		var_dump($problematicUrl);
	}
	$page++;
} while($page <= $problematicUrlsIterator->getLastPage());

// response (iterator)
[
	[
		'url' => 'https://webscraper.io/empty',
		'type' => 'empty',
	],
	[
		'url' => 'https://webscraper.io/failed',
		'type' => 'failed',
	],
	[
		'url' => 'https://webscraper.io/no-value',
		'type' => 'no_value',
	],
	[
	...
	],
]

const scrapingJobQuality = await client.getScrapingJobDataQuality(123);

{
	min_record_count: {
		got: 1,
		expected: 1,
		success: true // Specific data quality control indication
	},
	max_failed_pages_percent: {
		got: 0,
		expected: 5,
		success: true // Specific data quality control indication
	},
	max_empty_pages_percent: {
		got: 0,
		expected: 5,
		success: true // Specific data quality control indication
	},
	max_no_value_pages_percent: {
		got: 0,
		expected: 5,
		success: true // Specific data quality control indication
	},
	min_column_records: {
		title: {
			got: 100,
			expected: 95,
			success: true // Specific data quality control indication
		}
	},
	overall_data_quality_success: true // Global data quality control indication
}

$scrapingJob = $client->getScrapingJobDataQuality(123);

[
	'min_record_count' => [
		'got' => 1,
		'expected' => 1,
		'success' => true, // Specific data quality control indication
	],
	'max_failed_pages_percent' => [
		'got' => 0,
		'expected' => 5,
		'success' => true, // Specific data quality control indication
	],
	'max_empty_pages_percent' => [
		'got' => 0,
		'expected' => 5,
		'success' => true, // Specific data quality control indication
	],
	'max_no_value_pages_percent' => [
		'got' => 0,
		'expected' => 5,
		'success' => true, // Specific data quality control indication
	],
	'min_column_records' => [
		'title' => [
			'got' => 100,
			'expected' => 95,
			'success' => true, // Specific data quality control indication
		],
	],
	'overall_data_quality_success' => true, // Global data quality control indication
]

await client.deleteScrapingJob(500);

"ok"

$client->deleteScrapingJob(500);

"ok"

const info = await client.getAccountInfo();

{
	email: "user@example.com"
	firstname: "John"
	lastname: "Deere"
	page_credits: 500
}

$info = $client->getAccountInfo();

[
	'email' => 'user@example.com',
	'firstname' => 'John',
	'lastname' => 'Deere',
	'page_credits' => 500
]