10 use http\request\Factory as HttpRequestFactory;
11 use http\request\Pool as HttpRequestPool;
12 use http\request\Method as HttpRequestMethod;
13 use http\RequestException as HttpRequestException;
14 use http\SocketException as HttpSocketException;
19 ini_set('error_reporting', E_ALL);
20 ini_set('html_errors', 0);
22 class Pool extends HttpRequestPool
30 public final function run($factory, $urls_file = 'data/urls.txt', $cache_dir = 'HttpRequestPool_cache')
32 $this->factory = $factory;
33 $this->dir = (is_dir($cache_dir) or @mkdir($cache_dir)) ? $cache_dir : null;
35 foreach (array_map('trim', file($urls_file)) as $url) {
36 $this->all[$url] = $this->dir ? $this->dir .'/'. md5($url) : null;
42 public final function send()
45 $now = array_slice($this->all, 0, RMAX);
46 $this->rem = array_slice($this->all, RMAX);
52 foreach ($now as $url => $file) {
54 $this->factory->createRequest(
56 HttpRequestMethod::GET,
61 'connecttimeout' => TOUT,
62 'lastmodified' => is_file($file)?filemtime($file):0
68 while ($this->once()) {
70 throw new HttpSocketException;
75 protected final function once()
79 } catch (HttpRequestException $x) {
80 // a request may have thrown an exception,
81 // but it is still save to continue
82 echo $x->getMessage(), "\n";
85 foreach ($this->getFinishedRequests() as $r) {
89 $c = $r->getResponseCode();
90 $b = $r->getResponseBody();
92 printf("%d %s %d\n", $c, $u, strlen($b));
94 if ($c == 200 && $this->dir) {
95 file_put_contents($this->all[$u], $b);
98 if ($a = each($this->rem)) {
99 list($url, $file) = $a;
101 $this->factory->createRequest(
103 HttpRequestMethod::GET,
108 'connecttimeout' => TOUT,
109 'lastmodified' => is_file($file)?filemtime($file):0
119 define('GZIP', true);
124 $time = microtime(true);
125 $factory = new HttpRequestFactory(array("driver" => "curl", "requestPoolClass" => "Pool"));
126 $factory->createPool()->run($factory);
127 printf("Elapsed: %0.3fs\n", microtime(true)-$time);