13 ini_set('error_reporting', E_ALL);
14 ini_set('html_errors', 0);
16 class Pool extends \http\Curl\Client\Pool
24 public final function run($factory, $urls_file = 'data/urls.txt', $cache_dir = 'HttpRequestPool_cache')
26 $this->factory = $factory;
27 $this->dir = (is_dir($cache_dir) or @mkdir($cache_dir)) ? $cache_dir : null;
29 foreach (array_map('trim', file($urls_file)) as $url) {
30 $this->all[$url] = $this->dir ? $this->dir .'/'. md5($url) : null;
36 public final function send()
39 $now = array_slice($this->all, 0, RMAX);
40 $this->rem = array_slice($this->all, RMAX);
46 foreach ($now as $url => $file) {
48 $this->factory->createClient(
53 'connecttimeout' => TOUT,
54 'lastmodified' => is_file($file)?filemtime($file):0
56 )->setRequest(new http\Client\Request("GET", $url))
60 while ($this->once()) {
62 throw new http\Exception;
67 protected final function once()
71 } catch (http\Exception $x) {
72 // a request may have thrown an exception,
73 // but it is still save to continue
74 echo $x->getMessage(), "\n";
77 foreach ($this->getFinished() as $r) {
80 $u = $r->getRequest()->getRequestUrl();
81 $c = $r->getResponseMessage()->getResponseCode();
83 $b = $r->getResponseMessage()->getBody();
84 } catch (\Exception $e) {
85 echo $e->getMessage(), "\n";
89 printf("%d %s %d\n", $c, $u, strlen($b));
91 if ($c == 200 && $this->dir) {
92 file_put_contents($this->all[$u], $b);
95 if ($a = each($this->rem)) {
96 list($url, $file) = $a;
98 $this->factory->createClient(
103 'connecttimeout' => TOUT,
104 'lastmodified' => is_file($file)?filemtime($file):0
106 )->setRequest(new http\Client\Request("GET", $url))
114 define('GZIP', true);
119 $time = microtime(true);
120 $factory = new http\Client\Factory(array("driver" => "curl", "clientPoolClass" => "Pool"));
121 $factory->createPool()->run($factory);
122 printf("Elapsed: %0.3fs\n", microtime(true)-$time);