en:objects:harvestor:init:init

init

init($in_file,$proxy_file="",$process_js=true); - initialize the url collector

The function accepts parameters as input:

$in_file – path to the file with URLs that need to be collected

$proxy_file – path to the file with proxies that are used for collecting

$proceed_js – whether to process JS when collecting

After testing, the function returns the result of its work to the script :

true – successfully initialized

false – failed to initialize

PHP example:

<?php $xhe_host = "127.0.0.1:7022";
//+ cef
//connect the object to control the emulator, if not already connected
if(!isset($path))
  $path = "../../../Templates/xweb_human_emulator.php";
require($path);
 
//Start
echo "<hr> <font color = blue> webpage->".basename(__FILE__)."</font> <hr>";
 
//1
echo "1.Let's create a file with urls:";
$path = "test/in_file.urls";
echo $file_os->delete("test/in_file.urls")."";
echo $textfile->add_string_to_file("test/in_file.urls", "ya.ru
")." ";
echo $textfile->add_string_to_file("test/in_file.urls", "google.com
")." ";
echo $textfile->add_string_to_file("test/in_file.urls", "yandex.ru
")." ";
echo $textfile->add_string_to_file("test/in_file.urls", "vk.com
")." ";
echo $textfile->add_string_to_file("test/in_file.urls", "ttttttt
")." ";
echo $textfile->add_string_to_file("test/in_file.urls", "http://whoer.net
")." ";
 
//2
echo "
2.We initialize the collector with a file with urls: ";
echo $harvestor->init("test/in_file.urls")."";
 
//3
echo "
3.Start the collection and wait until the end: ";
echo $harvestor->start(false)."<br>";
while(!$harvestor->is_finished())
  echo ".".$harvestor->get_completed_count().".";
 
//4
echo "
4.Output the length of the resulting urls in the file: ";
$count = $textfile->get_lines_count($path);
for($i = 0; $i <$count; $i ++)
  echo strlen($harvestor->get_html($i))."";
 
//five
echo "
5.We initialize the collector with a file with urls, using a proxy and without JS: ";
echo $harvestor->init("test/in_file.urls", "test/proxies.txt", false)."";
 
//6
echo "
6.Start the collection and wait until the end: ";
echo $harvestor->start(false)."<br>";
while(!$harvestor->is_finished())
{
  echo ".".$harvestor->get_completed_count().".";
  sleep(1);
}
 
//7
echo "
7.Display the length of the resulting urls in the file: ";
$count = $textfile->get_lines_count($path);
for($i = 0; $i <$count; $i ++)
  echo strlen($harvestor->get_html($i))."";
 
//end
echo "<hr> <br>";
 
//Quit
$app->quit();
?>

Python example:

# Additional paths
import sys
sys.path.insert(0, '../../../Templates PY/')
 
xhe_host = "127.0.0.1:7022"
from xweb_human_emulator import *
 
# Start
echo("<hr> <font color = blue> ftp.xxxxxxxxx </font> <hr>")
 
# 1
echo("1.Let's create a file with urls:");
path = "test/in_file.urls";
echo(file_os.delete("test/in_file.urls"), "");
echo(textfile.add_string_to_file("test/in_file.urls", "ya.ru
")," ");
echo(textfile.add_string_to_file("test/in_file.urls", "google.com
")," ");
echo(textfile.add_string_to_file("test/in_file.urls", "yandex.ru
")," ");
echo(textfile.add_string_to_file("test/in_file.urls", "vk.com
")," ");
echo(textfile.add_string_to_file("test/in_file.urls", "ttttttt
")," ");
 
# 2
echo("
2.We initialize the collector with a file with urls: ");
echo(harvestor.init("test/in_file.urls"), "");
 
# 3
echo("
3.Start the collection and wait until the end: ");
echo(harvestor.start(false), "<br>");
while not harvestor.is_finished():
  echo("." + str(harvestor.get_completed_count()) + ".");
  sleep(1);
 
# 4
echo("
4.Display the length of the resulting urls in the file: ");
count = textfile.get_lines_count(path);
for i in range(0, count):
  echo(len(harvestor.get_html(i)), "")
 
# five
echo("
5.We initialize the collector with a file with urls: ");
echo(harvestor.init("test/in_file.urls", "test/proxies.txt", false), "");
 
# 6
echo("
6.Start the collection and wait until the end: ");
echo(harvestor.start(false), "<br>");
while not harvestor.is_finished():
  echo("." + str(harvestor.get_completed_count()) + ".");
  sleep(1);
 
# 7
echo("
7.Display the length of the resulting urls in the file: ");
count = textfile.get_lines_count(path);
for i in range(0, count):
  echo(len(harvestor.get_html(i)), "")
 
# end
echo("<hr> <br>")
 
# Quit
app.quit()

C# example:

#region using
 
using System;
using System.Diagnostics;
using System.Collections.Generic;
using System.Linq;
using System.IO;
using System.Text;
using System.Threading;
 
using XHE;
using XHE.XHE_DOM;
using XHE.XHE_System;
using XHE.XHE_Window;
using XHE.XHE_Web;
 
#endregion
 
 class Program: XHEScript
 {
static void Main(string [] args)
{
//init XHE
server = "127.0.0.1:7022";
InitXHE();
 
//Start
echo("<hr> <font color = blue> ftp.command </font> <hr>");
 
           //1
            echo("1.Let's create a file with urls:");
            string path = "test \ in_file.urls";
            echo(file_os.delete("test/in_file.urls") + "");
            echo(textfile.add_string_to_file("test \ in_file.urls", "ya.ru  n") + "");
            echo(textfile.add_string_to_file("test \ in_file.urls", "google.com  n") + "");
            echo(textfile.add_string_to_file("test \ in_file.urls", "yandex.ru  n") + "");
            echo(textfile.add_string_to_file("test \ in_file.urls", "vk.com  n") + "");
            echo(textfile.add_string_to_file("test \ in_file.urls", "ttttttt  n") + "");
 
           //2
            echo(" n2.We initialize the collector with a file with urls:");
            echo(harvestor.init("test \ in_file.urls") + "");
 
           //3
            echo(" n3.Start collection and wait for the end:");
            echo(harvestor.start(false) + "<br>");
            while(!harvestor.is_finished())
            {
              echo("." + harvestor.get_completed_count() + ".");
              sleep(1);
            }
 
           //4
            echo(" n4.Print the length of the received urls in the file:");
            int count = textfile.get_lines_count(path);
            for(int i = 0; i <count; i ++)
              echo(harvestor.get_html(i) .Length + "");
 
           //five
            echo(" n5.Initialize the collector with url file(using proxy and without JS processing):");
            echo(harvestor.init("test \ in_file.urls", "test \ proxies.txt", false) + "");
 
           //6
            echo(" n6.Start collection and wait until the end:");
            echo(harvestor.start(false) + "<br>");
            while(!harvestor.is_finished())
            {
              echo("." + harvestor.get_completed_count() + ".");
              sleep(1);
            }
 
           //7
            echo(" n7.Print the length of the received urls in the file:");
            count = textfile.get_lines_count(path);
            for(int i = 0; i <count; i ++)
              echo(harvestor.get_html(i) .Length + "");
 
//end
echo("<hr> <br>");
 
app.quit();
}
}

Java Script example:

//connect the object to control the emulator, if not already connected
xhe_host = "127.0.0.1:7022";
echo = require("../../../Templates JS/xweb_human_emulator.js");
 
//Start
echo("<hr> <font color = blue> ftp.command </font> <hr>");
 
//1
echo("1.Let's create a file with urls:");
path = "test/in_file.urls";
echo(file_os.delete("test/in_file.urls") + "");
echo(textfile.add_string_to_file("test/in_file.urls", "ya.ru  n") + "");
echo(textfile.add_string_to_file("test/in_file.urls", "google.com  n") + "");
echo(textfile.add_string_to_file("test/in_file.urls", "yandex.ru  n") + "");
echo(textfile.add_string_to_file("test/in_file.urls", "vk.com  n") + "");
echo(textfile.add_string_to_file("test/in_file.urls", "ttttttt  n") + "");
 
//2
echo(" n2.We initialize the collector with a file with urls:");
echo(harvestor.init("test/in_file.urls") + "");
 
//3
echo(" n3.Start collection and wait for the end:");
echo(harvestor.start(false) + "<br>");
while(!harvestor.is_finished())
  echo("." + harvestor.get_completed_count() + ".");
 
//4
echo(" n4.Print the length of the received urls in the file:");
count = textfile.get_lines_count(path);
for(i = 0; i <count; i ++)
  echo(harvestor.get_html(i) .length, "");
 
//five
echo(" n5.We initialize the collector with a file with urls:");
echo(harvestor.init("test/in_file.urls", "test/proxies.txt", false) + "");
 
//6
echo(" n6.Start collection and wait until the end:");
echo(harvestor.start(false) + "<br>");
while(!harvestor.is_finished())
  echo("." + harvestor.get_completed_count() + ".");
 
//7
echo(" n7.Print the length of the received urls in the file:");
count = textfile.get_lines_count(path);
for(i = 0; i <count; i ++)
  echo(harvestor.get_html(i) .length, "");
 
//end
echo("<hr> <br>");
 
//Quit
app.quit();