From: Michael Wallner Date: Tue, 12 May 2015 09:21:25 +0000 (+0200) Subject: add logging; fix caching X-Git-Url: https://git.m6w6.name/?a=commitdiff_plain;h=eb76e9bb9a39fe2725301f6cf6fc3cf29bbc4e00;p=pharext%2Fpharext.org add logging; fix caching --- diff --git a/app/Controller/Github.php b/app/Controller/Github.php index 0c38bd7..364f0e5 100644 --- a/app/Controller/Github.php +++ b/app/Controller/Github.php @@ -8,7 +8,7 @@ use app\Session; use app\Web; use http\QueryString; -use http\Url; +use http\Header; abstract class Github implements Controller { @@ -36,6 +36,15 @@ abstract class Github implements Controller "title" => "Github" ]); $this->app->getView()->registerFunction("check", [$this, "checkRepoHook"]); + + if (($header = $this->app->getRequest()->getHeader("Cache-Control", Header::class))) { + $params = $header->getParams(); + if (!empty($params["no-cache"])) { + $this->github->setMaxAge(0); + } elseif (!empty($params["max-age"])) { + $this->github->setMaxAge($params["max-age"]["value"]); + } + } } protected function checkToken() { @@ -50,7 +59,7 @@ abstract class Github implements Controller } /** - * Check if the pharext webhook is set for the repo and return its id + * Check if the pharext webhook is set for the repo and return it * @param object $repo * @return int hook id */ @@ -64,20 +73,4 @@ abstract class Github implements Controller } return null; } - - function createLinkGenerator($links) { - return function($which) use($links) { - if (!isset($links[$which])) { - if ($which !== "next" || !isset($links["last"])) { - return null; - } else { - $which = "last"; - } - } - $url = new Url($links[$which], null, 0); - $qry = new QueryString($url->query); - return $qry->getInt("page", 1); - }; - } - } diff --git a/app/Controller/Github/Callback.php b/app/Controller/Github/Callback.php index 496baf3..1b7f34f 100644 --- a/app/Controller/Github/Callback.php +++ b/app/Controller/Github/Callback.php @@ -33,7 +33,7 @@ class Callback extends Github $this->app->getRequest()->getQuery("state"), function($token) { $this->github->setToken($token->access_token); - $this->github->fetchUser($this->createUserCallback($token)); + $this->github->readAuthUser($this->createUserCallback($token)); })->send(); if (isset($this->session->returnto)) { $returnto = $this->session->returnto; diff --git a/app/Controller/Github/Index.php b/app/Controller/Github/Index.php index 935f249..c0838b7 100644 --- a/app/Controller/Github/Index.php +++ b/app/Controller/Github/Index.php @@ -8,7 +8,7 @@ class Index extends Github { function __invoke(array $args = null) { if ($this->checkToken()) { - $this->github->fetchRepos( + $this->github->listRepos( $this->app->getRequest()->getQuery("page"), [$this, "reposCallback"] )->send(); @@ -20,7 +20,7 @@ class Index extends Github $this->app->getView()->addData(compact("repos", "links")); foreach ($repos as $repo) { - $this->github->fetchHooks($repo->full_name, function($hooks) use($repo) { + $this->github->listHooks($repo->full_name, function($hooks) use($repo) { $repo->hooks = $hooks; }); } diff --git a/app/Controller/Github/Repo.php b/app/Controller/Github/Repo.php index 8caba2a..bf1358a 100644 --- a/app/Controller/Github/Repo.php +++ b/app/Controller/Github/Repo.php @@ -10,7 +10,7 @@ class Repo extends Github extract($args); $this->app->getView()->addData(compact("owner", "name")); if ($this->checkToken()) { - $this->github->fetchRepo( + $this->github->readRepo( "$owner/$name", [$this, "repoCallback"] )->send(); @@ -30,12 +30,12 @@ class Repo extends Github "title" => "Github: {$repo->name}" ]); settype($repo->tags, "object"); - $this->github->fetchHooks($repo->full_name, function($hooks) use($repo) { + $this->github->listHooks($repo->full_name, function($hooks) use($repo) { $repo->hooks = $hooks; }); - $this->github->fetchTags($repo->full_name, 1, $this->createTagsCallback($repo)); - $this->github->fetchReleases($repo->full_name, 1, $this->createReleasesCallback($repo)); - $this->github->fetchContents($repo->full_name, null, $this->createContentsCallback($repo)); + $this->github->listTags($repo->full_name, 1, $this->createTagsCallback($repo)); + $this->github->listReleases($repo->full_name, 1, $this->createReleasesCallback($repo)); + $this->github->readContents($repo->full_name, null, $this->createContentsCallback($repo)); } function createReleasesCallback($repo) { diff --git a/app/Controller/Github/RepoHook.php b/app/Controller/Github/RepoHook.php index 6733b3f..023763a 100644 --- a/app/Controller/Github/RepoHook.php +++ b/app/Controller/Github/RepoHook.php @@ -3,6 +3,7 @@ namespace app\Controller\Github; use app\Controller\Github; +use app\Github\API\Hooks\ListHooks; class RepoHook extends Github { @@ -34,20 +35,22 @@ class RepoHook extends Github function addHook($owner, $repo) { $hook_conf = $this->app->getRequest()->getForm(); - $call = $this->github->createRepoHook("$owner/$repo", $hook_conf, function($hook) use($owner, $repo, &$call) { - $call->dropFromCache(); - $this->redirectBack("$owner/$repo"); - }); - $call->send(); + $this->github->createRepoHook("$owner/$repo", $hook_conf, function($hook) use($owner, $repo) { + $call = new ListHooks($this->github, ["repo" => "$owner/$repo", "fresh" => true]); + $call(function($hooks, $links) use($owner, $repo, $call) { + $call->saveToCache([$hooks, $links]); + $this->redirectBack("$owner/$repo"); + }); + })->send(); } function updateHook($owner, $repo) { - $this->github->fetchRepo("$owner/$repo", function($repo) { - $call = $this->github->fetchHooks($repo->full_name, function($hooks, $links) use($repo, &$call) { + $this->github->readRepo("$owner/$repo", function($repo) { + $call = $this->github->listHooks($repo->full_name, function($hooks, $links) use($repo, &$call) { $repo->hooks = $hooks; if (($hook = $this->checkRepoHook($repo))) { $hook_conf = $this->app->getRequest()->getForm(); - $this->github->updateRepoHook($repo->full_name, $hook->id, $hook_conf, function($changed_hook) use($repo, $hook, $hooks, $links, $call) { + $this->github->updateRepoHook($repo->full_name, $hook->id, $hook_conf, function($changed_hook) use($repo, $hook, $hooks, $links, &$call) { foreach ($changed_hook as $key => $val) { $hook->$key = $val; } @@ -60,11 +63,11 @@ class RepoHook extends Github } function delHook($owner, $repo) { - $this->github->fetchRepo("$owner/$repo", function($repo) { - $call = $this->github->fetchHooks($repo->full_name, function($hooks) use($repo, &$call) { + $this->github->readRepo("$owner/$repo", function($repo) { + $call = $this->github->listHooks($repo->full_name, function($hooks) use($repo, &$call) { $repo->hooks = $hooks; if (($hook = $this->checkRepoHook($repo))) { - $this->github->deleteRepoHook($repo->full_name, $hook->id, function() use($repo, $call) { + $this->github->deleteRepoHook($repo->full_name, $hook->id, function() use($repo, &$call) { $call->dropFromCache(); $this->redirectBack($repo->full_name); }); diff --git a/app/Github/API.php b/app/Github/API.php index 1023769..0f63a60 100644 --- a/app/Github/API.php +++ b/app/Github/API.php @@ -12,6 +12,8 @@ use http\Client; use http\QueryString; use http\Url; +use Psr\Log\LoggerInterface; + class API { /** @@ -33,13 +35,42 @@ class API * @var merry\Config */ private $config; + + /** + * @var int + */ + private $maxAge; + + /** + * @var \Psr\Log\LoggerInterface; + */ + private $logger; - function __construct(Config $config, Storage $tokens = null, Storage $cache = null) { + function __construct(Config $config, LoggerInterface $logger, Storage $tokens = null, Storage $cache = null) { + $this->logger = $logger; $this->config = $config; $this->client = new Client; + $this->client->attach(new ClientObserver($logger)); $this->tokens = $tokens ?: new Storage\Session; $this->cache = $cache; } + + /** + * Set maximum acceptable age of cache items + * @param int $seconds + */ + function setMaxAge($seconds) { + $this->maxAge = $seconds; + return $this; + } + + function getMaxAge() { + return $this->maxAge; + } + + function getLogger() { + return $this->logger; + } function getConfig() { return $this->config; @@ -66,16 +97,19 @@ class API } function setToken($token) { - $this->tokens->set("access_token", $token, - $this->config->storage->token->ttl); + $this->tokens->set("access_token", new Storage\Item( + $token, + $this->config->storage->token->ttl + )); } function getToken() { - if ($this->tokens->get("access_token", $token, $ttl, true)) { - return $token; + if ($this->tokens->get("access_token", $token, true)) { + return $token->getValue(); } - if (isset($ttl)) { - throw new Exception\TokenExpired($ttl); + if (isset($token)) { + $this->logger->notice("Token expired", $token); + throw new Exception\TokenExpired($token->getLTL()); } throw new Exception\TokenNotSet; } @@ -86,7 +120,7 @@ class API function getAuthUrl($callback_url) { $state = base64_encode(openssl_random_pseudo_bytes(24)); - $this->tokens->set("state", $state, 5*60); + $this->tokens->set("state", new Storage\Item($state, 5*60)); $param = [ "state" => $state, "client_id" => $this->config->client->id, @@ -99,14 +133,16 @@ class API } function fetchToken($code, $state, callable $callback) { - if (!$this->tokens->get("state", $orig_state, $ttl, true)) { - if (isset($ttl)) { - throw new Exception\StateExpired($ttl); + if (!$this->tokens->get("state", $orig_state, true)) { + if (isset($orig_state)) { + $this->logger->notice("State expired", $orig_state); + throw new Exception\StateExpired($orig_state->getLTL()); } throw new Exception\StateNotSet; } - if ($state !== $orig_state) { - throw new Exception\StateMismatch($orig_state, $state); + if ($state !== $orig_state->getValue()) { + $this->logger->warning("State mismatch", compact("state", "orig_state")); + throw new Exception\StateMismatch($orig_state->getValue(), $state); } $call = new API\Users\ReadAuthToken($this, [ @@ -117,37 +153,37 @@ class API return $call($callback); } - function fetchUser(callable $callback) { + function readAuthUser(callable $callback) { $call = new API\Users\ReadAuthUser($this); return $call($callback); } - function fetchRepos($page, callable $callback) { + function listRepos($page, callable $callback) { $call = new API\Repos\ListRepos($this, compact("page")); return $call($callback); } - function fetchRepo($repo, callable $callback) { + function readRepo($repo, callable $callback) { $call = new API\Repos\ReadRepo($this, compact("repo")); return $call($callback); } - function fetchHooks($repo, callable $callback) { + function listHooks($repo, callable $callback) { $call = new API\Hooks\ListHooks($this, compact("repo")); return $call($callback); } - function fetchReleases($repo, $page, callable $callback) { + function listReleases($repo, $page, callable $callback) { $call = new API\Releases\ListReleases($this, compact("repo", "page")); return $call($callback); } - function fetchTags($repo, $page, callable $callback) { + function listTags($repo, $page, callable $callback) { $call = new API\Tags\ListTags($this, compact("repo", "page")); return $call($callback); } - function fetchContents($repo, $path, callable $callback) { + function readContents($repo, $path, callable $callback) { $call = new API\Repos\ReadContents($this, compact("repo", "path")); return $call($callback); } diff --git a/app/Github/API/Call.php b/app/Github/API/Call.php index 32bf1cb..d4afd19 100644 --- a/app/Github/API/Call.php +++ b/app/Github/API/Call.php @@ -3,6 +3,7 @@ namespace app\Github\API; use app\Github\API; +use app\Github\Storage\Item; use http\QueryString; use http\Url; use merry\Config; @@ -96,17 +97,33 @@ abstract class Call function getCacheKey() { $args = $this->args; unset($args["fresh"]); + if (isset($args["page"]) && !strcmp($args["page"], "1")) { + unset($args["page"]); + } ksort($args); return sprintf("%s:%s:%s", $this->api->getToken(), $this, new QueryString($args)); } - function readFromCache(array &$cached = null, &$ttl = null) { - if (empty($this->args["fresh"]) && ($cache = $this->api->getCacheStorage())) { - $key = $this->getCacheKey(); - return $cache->get($key, $cached, $ttl); + function readFromCache(array &$value = null) { + if (!empty($this->args["fresh"])) { + return false; + } + if (!($cache = $this->api->getCacheStorage())) { + return false; + } + if (!strlen($key = $this->getCacheKey())) { + return false; + } + if (!$cache->get($key, $cached)) { + return false; + } + if (null !== $this->api->getMaxAge() && $cached->getAge() > $this->api->getMaxAge()) { + return false; } - return false; + $this->api->getLogger()->debug("Cache-Hit: $this", $this->args); + $value = $cached->getValue(); + return true; } function saveToCache(array $fresh) { @@ -118,7 +135,7 @@ abstract class Call } $key = $this->getCacheKey(); - $cache->set($key, $fresh, $ttl); + $cache->set($key, new Item($fresh, $ttl)); } } diff --git a/app/Github/API/Hooks/DeleteHook.php b/app/Github/API/Hooks/DeleteHook.php index 311e64d..a94f285 100644 --- a/app/Github/API/Hooks/DeleteHook.php +++ b/app/Github/API/Hooks/DeleteHook.php @@ -18,7 +18,7 @@ class DeleteHook extends Call if ($response->getResponseCode() >= 400) { throw new RequestException($response); } - $callback($json); + $callback(); return true; }); } diff --git a/app/Github/API/Users/ReadAuthToken.php b/app/Github/API/Users/ReadAuthToken.php index 4430b45..bc1f03b 100644 --- a/app/Github/API/Users/ReadAuthToken.php +++ b/app/Github/API/Users/ReadAuthToken.php @@ -22,4 +22,8 @@ class ReadAuthToken extends Call return true; }); } + + function getCacheKey() { + return null; + } } diff --git a/app/Github/ClientObserver.php b/app/Github/ClientObserver.php new file mode 100644 index 0000000..89e01d8 --- /dev/null +++ b/app/Github/ClientObserver.php @@ -0,0 +1,41 @@ +logger = $logger; + } + + function update(SplSubject $client, Request $request = null, $progress = null) { + switch ($progress->info) { + case "start": + if (!$progress->started) { + $message = sprintf("API-Shot: start %s %s", $request->getRequestMethod(), $request->getRequestUrl()); + $this->logger->debug($message); + } + break; + case "finished": + $response = $client->getResponse($request); + $message = sprintf("API-Shot: finished [%d] %s %s", $response->getResponseCode(), $request->getRequestMethod(), $request->getRequestUrl()); + if ($response->getResponseCode() >= 400 || $response->getTransferInfo("error")) { + $this->logger->error($message, (array) $response->getTransferInfo()); + } else { + $this->logger->info($message); + } + break; + default: + break; + } + } +} diff --git a/app/Github/Exception/RequestException.php b/app/Github/Exception/RequestException.php index 49ea1ae..c0a3d33 100644 --- a/app/Github/Exception/RequestException.php +++ b/app/Github/Exception/RequestException.php @@ -47,10 +47,15 @@ class RequestException extends \Exception implements Exception $errors = "JSON errors:\n"; foreach ($this->errors as $error) { - $errors .= sprintf($reasons[$error->code], $error->resource, $error->field); + if ($error->code === "custom") { + $errors .= $error->message . "\n"; + } else { + $errors .= sprintf($reasons[$error->code], $error->resource, $error->field); + } } return $errors; } + function __toString() { return parent::__toString() . "\n". $this->getErrorsAsString(); } diff --git a/app/Github/Logger.php b/app/Github/Logger.php new file mode 100644 index 0000000..d90e824 --- /dev/null +++ b/app/Github/Logger.php @@ -0,0 +1,22 @@ +github->log; + parent::__construct($channel); + foreach ($config->log->$channel as $logger) { + $reflection = new \ReflectionClass("Monolog\\Handler\\" . $logger->handler); + if (!empty($logger->args)) { + $handler = $reflection->newInstanceArgs($logger->args->toArray()); + } else { + $handler = $reflection->newInstance(); + } + $this->pushHandler($handler); + } + } +} diff --git a/app/Github/Storage.php b/app/Github/Storage.php index 3855107..bf572b4 100644 --- a/app/Github/Storage.php +++ b/app/Github/Storage.php @@ -4,7 +4,7 @@ namespace app\Github; interface Storage { - function set($key, $val, $ttl = null); - function get($key, &$val = null, &$ttl = null, $update = false); + function set($key, Storage\Item $item); + function get($key, Storage\Item &$item = null, $update = false); function del($key); } diff --git a/app/Github/Storage/Item.php b/app/Github/Storage/Item.php new file mode 100644 index 0000000..5bd6bbe --- /dev/null +++ b/app/Github/Storage/Item.php @@ -0,0 +1,63 @@ +value = $value; + $this->ttl = $ttl; + $this->time = $time ?: time(); + } + + static function __set_state(array $state) { + return new static( + isset($state["value"]) ? $state["value"] : null, + isset($state["ttl"]) ? $state["ttl"] : null, + isset($state["time"]) ? $state["time"] : null + ); + } + + function toArray() { + return get_object_vars($this); + } + + function getTimestamp() { + return $this->time; + } + + function setTimestamp($ts = null) { + $this->time = $ts ?: time(); + return $this; + } + + function getTTL() { + return $this->ttl; + } + + function setTTL($ttl = null) { + $this->ttl = $ttl; + return $this; + } + + function getAge($from = null) { + return ($from ?: time()) - $this->time; + } + + function getLTL($from = null) { + return $this->ttl - $this->getAge($from); + } + + function getValue() { + return $this->value; + } + + function setValue($value = null) { + $this->value = $value; + return $this; + } +} diff --git a/app/Github/Storage/Memcache.php b/app/Github/Storage/Memcache.php index 1fd189d..9d76a75 100644 --- a/app/Github/Storage/Memcache.php +++ b/app/Github/Storage/Memcache.php @@ -22,7 +22,7 @@ class Memcache implements Storage return sprintf("%s:%s", $this->ns, $key); } - function get($key, &$val = null, &$ltl = null, $update = false) { + function get($key, Item &$item = null, $update = false) { if (!$item = $this->mc->get($this->key($key))) { return false; } @@ -31,28 +31,21 @@ class Memcache implements Storage $ttl = $item->ttl; $set = $item->time; - if (!isset($ttl)) { + if (null === $item->getTTL()) { return true; } - $now = time(); - $ltl = $ttl - ($now - $set); - if ($ltl >= 0) { + if ($item->getLTL() >= 0) { if ($update) { - $item->time = time(); - $this->mc->set($this->key($key), $item, $ttl + 60*60*24); + $item->setTimestamp(); + $this->mc->set($this->key($key), $item, $item->getTTL() + 60*60*24); } return true; } return false; } - function set($key, $val, $ttl = null) { - $item = new Memcache\Item([ - "value" => $val, - "ttl" => $ttl, - "time" => isset($ttl) ? time() : null - ]); - $this->mc->set($this->key($key), $item, isset($ttl) ? $ttl + 60*60*24 : 0); + function set($key, Item $item) { + $this->mc->set($this->key($key), $item, (null !== $item->getTTL()) ? $item->getTTL() + 60*60*24 : 0); return $this; } @@ -60,19 +53,3 @@ class Memcache implements Storage $this->mc->delete($this->key($key)); } } - -namespace app\Github\Storage\Memcache; - -class Item -{ - public $value; - public $time; - public $ttl; - - function __construct(array $data) { - foreach ($data as $key => $val) { - $this->$key = $val; - } - } -} - diff --git a/app/Github/Storage/Redis.php b/app/Github/Storage/Redis.php index 13d641a..c971687 100644 --- a/app/Github/Storage/Redis.php +++ b/app/Github/Storage/Redis.php @@ -23,41 +23,29 @@ class Redis implements Storage return sprintf("%s:%s", $this->ns, $key); } - function get($key, &$val = null, &$ltl = null, $update = false) { + function get($key, Item &$item = null, $update = false) { if (!$item = $this->rd->get($this->key($key))) { - header("Cache-Item: ".serialize($item), false); return false; } - $val = $item->value; - $ttl = $item->ttl; - $set = $item->time; - - if (!isset($ttl)) { + if (null === $item->getTTL()) { return true; } - $now = time(); - $ltl = $ttl - ($now - $set); - if ($ltl >= 0) { + if ($item->getLTL() >= 0) { if ($update) { - $item->time = time(); - $this->rd->setex($this->key($key), $ttl + 60*60*24, $item); + $item->setTimestamp(); + $this->rd->setex($this->key($key), $item->getTTL() + 60*60*24, $item); } return true; } return false; } - function set($key, $val, $ttl = null) { - $item = new Redis\Item([ - "value" => $val, - "ttl" => $ttl, - "time" => isset($ttl) ? time() : null - ]); - if (isset($ttl)) { + function set($key, Item $item) { + if (null === $item->getTTL()) { $this->rd->set($this->key($key), $item); } else { - $this->rd->setex($this->key($key), $ttl + 60*60*24, $item); + $this->rd->setex($this->key($key), $item->getTTL() + 60*60*24, $item); } return $this; } @@ -66,19 +54,3 @@ class Redis implements Storage $this->rd->delete($this->key($key)); } } - -namespace app\Github\Storage\Redis; - -class Item -{ - public $value; - public $time; - public $ttl; - - function __construct(array $data) { - foreach ($data as $key => $val) { - $this->$key = $val; - } - } -} - diff --git a/app/Github/Storage/Session.php b/app/Github/Storage/Session.php index 943e975..f16cdb3 100644 --- a/app/Github/Storage/Session.php +++ b/app/Github/Storage/Session.php @@ -12,24 +12,22 @@ class Session implements Storage $this->ns = $ns; } - function set($key, $val, $ttl = null) { - $_SESSION[$this->ns][$key] = [$val, $ttl, isset($ttl) ? time() : null]; + function set($key, Item $item) { + $_SESSION[$this->ns][$key] = $item; return $this; } - function get($key, &$val = null, &$ltl = null, $update = false) { + function get($key, Item &$item = null, $update = false) { if (!isset($_SESSION[$this->ns][$key])) { return false; } - list($val, $ttl, $set) = $_SESSION[$this->ns][$key]; - if (!isset($ttl)) { + $item = $_SESSION[$this->ns][$key]; + if (null === $item->getTTL()) { return true; } - $now = time(); - $ltl = $ttl - ($now - $set); - if ($ltl >= 0) { + if ($item->getLTL() >= 0) { if ($update) { - $_SESSION[$this->ns][$key][2] = $now; + $item->setTimestamp(); } return true; } diff --git a/app/bootstrap/github.php b/app/bootstrap/github.php index 0ec014e..30bd96d 100644 --- a/app/bootstrap/github.php +++ b/app/bootstrap/github.php @@ -16,8 +16,10 @@ $injector->share(Github\API::class) $config->$basic->auth->toArray(), 0); } + // FIXME: configure through app.ini return new Github\API( $config->github + ,new Github\Logger($config) ,new Github\Storage\Session("gh-tokens") #,new Github\Storage\Memcache("gh-cache") ,new Github\Storage\Redis("gh-cache") diff --git a/config/app.ini b/config/app.ini index ec01dbf..201de74 100644 --- a/config/app.ini +++ b/config/app.ini @@ -17,6 +17,8 @@ github.storage.cache.hooks.ttl = 3600 github.storage.cache.tags.ttl = 3600 github.storage.cache.releases.ttl = 3600 +github.log = github + session.use_cookies = 1 session.use_only_cookies = 1 session.use_strict_mode = 1 @@ -30,6 +32,11 @@ session.cache_expire = 0 pq.flags = 0 pq.dsn = "user=pharext host=localhost" +log.github.streamhandler.handler = StreamHandler +log.github.streamhandler.args[] = ../logs/github.log +; Logger::DEBUG == 100 +log.github.streamhandler.args[] = 100 + [localhost : production] github.hook.url = https://pharext.ngrok.io/src/pharext.org.git/public/github/hook