10 use http\request\Factory as HttpRequestFactory;
11 use http\request\Pool as HttpRequestPool;
12 use http\Exception as HttpRequestException;
13 use http\Exception as HttpSocketException;
18 ini_set('error_reporting', E_ALL);
19 ini_set('html_errors', 0);
21 class Pool extends HttpRequestPool
29 public final function run($factory, $urls_file = 'data/urls.txt', $cache_dir = 'HttpRequestPool_cache')
31 $this->factory = $factory;
32 $this->dir = (is_dir($cache_dir) or @mkdir($cache_dir)) ? $cache_dir : null;
34 foreach (array_map('trim', file($urls_file)) as $url) {
35 $this->all[$url] = $this->dir ? $this->dir .'/'. md5($url) : null;
41 public final function send()
44 $now = array_slice($this->all, 0, RMAX);
45 $this->rem = array_slice($this->all, RMAX);
51 foreach ($now as $url => $file) {
53 $this->factory->createRequest(
60 'connecttimeout' => TOUT,
61 'lastmodified' => is_file($file)?filemtime($file):0
67 while ($this->once()) {
69 throw new HttpSocketException;
74 protected final function once()
78 } catch (HttpRequestException $x) {
79 // a request may have thrown an exception,
80 // but it is still save to continue
81 echo $x->getMessage(), "\n";
84 foreach ($this->getFinishedRequests() as $r) {
88 $c = $r->getResponseCode();
90 $b = $r->getResponseBody();
91 } catch (\Exception $e) {
92 echo $e->getMessage(), "\n";
96 printf("%d %s %d\n", $c, $u, strlen($b));
98 if ($c == 200 && $this->dir) {
99 file_put_contents($this->all[$u], $b);
102 if ($a = each($this->rem)) {
103 list($url, $file) = $a;
105 $this->factory->createRequest(
112 'connecttimeout' => TOUT,
113 'lastmodified' => is_file($file)?filemtime($file):0
123 define('GZIP', true);
128 $time = microtime(true);
129 $factory = new HttpRequestFactory(array("driver" => "curl", "requestPoolClass" => "Pool"));
130 $factory->createPool()->run($factory);
131 printf("Elapsed: %0.3fs\n", microtime(true)-$time);