10 use http\request\Factory as HttpRequestFactory;
11 use http\request\Pool as HttpRequestPool;
12 use http\request\Method as HttpRequestMethod;
13 use http\Exception as HttpRequestException;
14 use http\Exception as HttpSocketException;
19 ini_set('error_reporting', E_ALL);
20 ini_set('html_errors', 0);
22 class Pool extends HttpRequestPool
30 public final function run($factory, $urls_file = 'data/urls.txt', $cache_dir = 'HttpRequestPool_cache')
32 $this->factory = $factory;
33 $this->dir = (is_dir($cache_dir) or @mkdir($cache_dir)) ? $cache_dir : null;
35 foreach (array_map('trim', file($urls_file)) as $url) {
36 $this->all[$url] = $this->dir ? $this->dir .'/'. md5($url) : null;
42 public final function send()
45 $now = array_slice($this->all, 0, RMAX);
46 $this->rem = array_slice($this->all, RMAX);
52 foreach ($now as $url => $file) {
54 $this->factory->createRequest(
56 HttpRequestMethod::GET,
61 'connecttimeout' => TOUT,
62 'lastmodified' => is_file($file)?filemtime($file):0
68 while ($this->once()) {
70 throw new HttpSocketException;
75 protected final function once()
79 } catch (HttpRequestException $x) {
80 // a request may have thrown an exception,
81 // but it is still save to continue
82 echo $x->getMessage(), "\n";
85 foreach ($this->getFinishedRequests() as $r) {
89 $c = $r->getResponseCode();
91 $b = $r->getResponseBody();
92 } catch (\Exception $e) {
93 echo $e->getMessage(), "\n";
97 printf("%d %s %d\n", $c, $u, strlen($b));
99 if ($c == 200 && $this->dir) {
100 file_put_contents($this->all[$u], $b);
103 if ($a = each($this->rem)) {
104 list($url, $file) = $a;
106 $this->factory->createRequest(
108 HttpRequestMethod::GET,
113 'connecttimeout' => TOUT,
114 'lastmodified' => is_file($file)?filemtime($file):0
124 define('GZIP', true);
129 $time = microtime(true);
130 $factory = new HttpRequestFactory(array("driver" => "curl", "requestPoolClass" => "Pool"));
131 $factory->createPool()->run($factory);
132 printf("Elapsed: %0.3fs\n", microtime(true)-$time);